Details
-
Bug
-
Resolution: Done
-
Major - P3
-
2.0.2
-
None
-
Distributor ID: Ubuntu
Description: Ubuntu 10.04.3 LTS
Release: 10.04
Codename: lucid
Linux data2 2.6.32-33-generic #72-Ubuntu SMP Fri Jul 29 21:07:13 UTC 2011 x86_64 GNU/Linux
-
Linux
Description
mongodb hang this and last weekend when doing mapreduce at the very
early stage. Following is the related log (i think), the full log is
at https://gist.github.com/2395867
The db is fully locked without yield and any collection related operations hang. Deadlock maybe?
The mapreduce code should be fine. After i restart the mongodb and
rerun the mapreduce, it finished in less than 10 minutes.
Following is the log and code. Unfortunately, the mongodb log was lost when i restart mongodb.
mongostat --port 27018 5
|
connected to: 127.0.0.1:27018
|
insert query update delete getmore command flushes mapped vsize res faults locked % idx miss % qr|qw ar|aw netIn netOut conn set repl time
|
0 0 0 0 0 1 0 75g 151g 9.1g 0 0 0 5|3 5|2 113b 424b 16 rs_default M 08:58:12
|
0 0 0 0 0 1 0 75g 151g 9.1g 0 0 0 5|3 5|2 163b 480b 16 rs_default M 08:58:17
|
|
|
PRIMARY> db.currentOp()
|
{
|
"inprog" : [
|
{
|
"opid" : 15710450,
|
"active" : true,
|
"lockType" : "read",
|
"waitingForLock" : true,
|
"secs_running" : 17965,
|
"op" : "getmore",
|
"ns" : "local.oplog.rs",
|
"query" : {
|
|
|
},
|
"client" : "192.168.0.21:55171",
|
"desc" : "conn",
|
"threadId" : "0x7f1e8d1f3700",
|
"connectionId" : 5,
|
"numYields" : 0
|
},
|
{
|
"opid" : 15710430,
|
"active" : true,
|
"lockType" : "read",
|
"waitingForLock" : false,
|
"secs_running" : 17966,
|
"op" : "query",
|
"ns" : "app_wowsearch.events.day",
|
"query" : {
|
"$msg" : "query not recording (too large)"
|
},
|
"client" : "127.0.0.1:33900",
|
"desc" : "conn",
|
"threadId" : "0x7f1e8daf5700",
|
"connectionId" : 19963,
|
"msg" : "m/r: (1/3) emit phase 0/835941 0%",
|
"progress" : {
|
"done" : 0,
|
"total" : 835941
|
},
|
"numYields" : 3
|
},
|
|
|
|
|
db.serverStatus()
|
{
|
"host" : "data2:27018",
|
"version" : "2.0.2",
|
"process" : "mongod",
|
"uptime" : 602434,
|
"uptimeEstimate" : 594271,
|
"localTime" : ISODate("2012-04-16T00:59:47.391Z"),
|
"globalLock" : {
|
"totalTime" : 602434261059,
|
"lockTime" : 56141056582,
|
"ratio" : 0.09319034492379538,
|
"currentQueue" : {
|
"total" : 8,
|
"readers" : 5,
|
"writers" : 3
|
},
|
"activeClients" : {
|
"total" : 7,
|
"readers" : 5,
|
"writers" : 2
|
}
|
},
|
Here is the map reduce code, a few utility functions are not provided. If you want them, I can provide too.
function rollup_cycle_users(cycle, firstday) { //{{{
|
var m = function() { //{{{
|
var l = this.l || {};
|
|
|
var mk = {id: this.id, l: {}};
|
var keys = ['s', 'rt', 'rc', 'rf', 'va'];
|
keys.forEach(function(k) {
|
mk.l[k] = l[k] || null;
|
});
|
|
|
var mv = extend(copy_obj(this, ['model', 'ch', 'city', 'province', 'avn', 'resolution']),
|
{
|
pvs: 0,
|
downloads: 0,
|
searchs: 0,
|
video_pls: 0,
|
days: {},
|
});
|
var e2dayname = {
|
search: 'sedays',
|
download: 'dldays',
|
pv: 'pvdays',
|
video_play: 'vpdays',
|
};
|
switch(this.e){
|
case 'download' : mv.downloads += this.c || 1; break;
|
case 'pv' : mv.pvs += this.c || 1; break;
|
case 'search' : mv.searchs += this.c || 1; break;
|
case 'video_play' : mv.video_pls += this.c || 1; break;
|
}
|
var day_name = e2dayname[this.e];
|
if (day_name) {
|
var d = 1 + (this.ts.getTime() - base_ts.getTime()) / (24 * 3600000); // start from 1
|
mv.days[day_name] = {};
|
mv.days[day_name][d] = true;
|
emit(mk, mv);
|
}
|
}; //}}}
|
|
|
var r = function(key, emits) { //{{{
|
var total0 = {downloads: 0, pvs: 0, video_pls:0, searchs:0, days: {}};
|
if (emits.length == 0) {
|
return total0;
|
}
|
var total = extend(emits[0], total0);
|
|
|
var tdays = total.days;
|
emits.forEach(function(emit) {
|
for (day_name in emit.days) {
|
tdays[day_name] = extend(tdays[day_name] || {}, emit.days[day_name]);
|
tdays[day_name] = tdays[day_name] || {};
|
for (d in emit.days[day_name]) {
|
tdays[day_name][d] = true;
|
}
|
}
|
total.downloads += emit.downloads;
|
total.pvs += emit.pvs;
|
total.video_pls += emit.video_pls;
|
total.searchs += emit.searchs;
|
});
|
return total;
|
}; //}}}
|
|
|
var f = function(key, total) { //{{{
|
function count(days) {
|
days = days || [];
|
return days.length;
|
}
|
function max_continuous_count(days) { //{{{
|
days = days || [];
|
if (!days.length) return 0;
|
|
|
var max_ccount = 0;
|
var ccount = 0;
|
days = days.concat([-1]); // guard
|
for (i in days) {
|
if (days[i] - days[i-1] != 1 || i == 0) {
|
max_ccount = Math.max(max_ccount, ccount);
|
ccount = 1;
|
}
|
else {
|
ccount += 1;
|
}
|
}
|
|
|
return max_ccount;
|
} //}}}
|
function max_delta(days) {
|
deltas = delta(days);
|
if (deltas.length == 0) return 0;
|
return Math.max.apply(null, deltas);
|
}
|
function min_delta(days) {
|
deltas = delta(days);
|
if (deltas.length == 0) return 0;
|
return Math.min.apply(null, deltas);
|
}
|
|
|
|
|
for (d in total.days) {
|
total.days[d] = keys(total.days[d]).sort(function(a, b) {return a-b;});
|
}
|
var days2al = {pvdays: 'pv', dldays: 'dl', vpdays: 'vp', sedays: 'se'}; //detail in map function 访问,下载,播放,搜索天数
|
|
|
var pvdays = total.days.pvdays || [];
|
var dldays = total.days.dldays || [];
|
aldays = {};
|
for (cdi in total.days) {
|
cd = total.days[cdi];
|
if (!cd) continue;
|
aldays[days2al[cdi]] = {
|
d: count(cd),
|
contd: max_continuous_count(cd),
|
mindd: min_delta(cd),
|
maxdd: max_delta(cd),
|
};
|
}
|
|
|
var min_dl_day = -1;
|
if (dldays.length) {
|
min_dl_day = Math.min(dldays);
|
}
|
dlpvdays = filter(pvdays, function(d) {
|
return d > min_dl_day;
|
});
|
dlpvd = count(dlpvdays);
|
|
|
total = extend(total, {dlpvd: dlpvd});
|
total = extend(total, aldays);
|
|
|
return total;
|
} //}}}
|
|
|
var cycle2delta = {
|
week: '7d',
|
month: '1m',
|
};
|
|
|
var colname = get_scale_col_name(cycle, firstday);
|
var mr_colname = 'mr.' + colname;
|
var ts0 = get_china_day(firstday);
|
var ts1 = date_add(ts0, cycle2delta[cycle]);
|
var query = {ts: {$gte: ts0, $lt: ts1}}; // [ts0, ts1)
|
db.events.day.mapReduce(
|
m, r, {
|
verbose: true,
|
out: mr_colname,
|
finalize: f,
|
query: query,
|
scope: {
|
base_ts: ts0,
|
copy_obj: copy_obj,
|
filter: filter,
|
extend: extend,
|
delta: delta,
|
keys: keys,
|
},
|
jsMode: true,
|
}
|
);
|
|
|
db[mr_colname].find().forEach(function(doc) {
|
var ndoc = extend(doc._id, doc.value);
|
db[colname].insert(ndoc);
|
});
|
|
|
} //}}}
|