-
Type:
Bug
-
Resolution: Done
-
Priority:
Major - P3
-
Affects Version/s: 2.0.2
-
Component/s: JavaScript, MapReduce
-
None
-
Environment:Distributor ID: Ubuntu
Description: Ubuntu 10.04.3 LTS
Release: 10.04
Codename: lucid
Linux data2 2.6.32-33-generic #72-Ubuntu SMP Fri Jul 29 21:07:13 UTC 2011 x86_64 GNU/Linux
-
Linux
-
None
-
3
-
None
-
None
-
None
-
None
-
None
-
None
mongodb hang this and last weekend when doing mapreduce at the very
early stage. Following is the related log (i think), the full log is
at https://gist.github.com/2395867
The db is fully locked without yield and any collection related operations hang. Deadlock maybe?
The mapreduce code should be fine. After i restart the mongodb and
rerun the mapreduce, it finished in less than 10 minutes.
Following is the log and code. Unfortunately, the mongodb log was lost when i restart mongodb.
mongostat --port 27018 5 connected to: 127.0.0.1:27018 insert query update delete getmore command flushes mapped vsize res faults locked % idx miss % qr|qw ar|aw netIn netOut conn set repl time 0 0 0 0 0 1 0 75g 151g 9.1g 0 0 0 5|3 5|2 113b 424b 16 rs_default M 08:58:12 0 0 0 0 0 1 0 75g 151g 9.1g 0 0 0 5|3 5|2 163b 480b 16 rs_default M 08:58:17 PRIMARY> db.currentOp() { "inprog" : [ { "opid" : 15710450, "active" : true, "lockType" : "read", "waitingForLock" : true, "secs_running" : 17965, "op" : "getmore", "ns" : "local.oplog.rs", "query" : { }, "client" : "192.168.0.21:55171", "desc" : "conn", "threadId" : "0x7f1e8d1f3700", "connectionId" : 5, "numYields" : 0 }, { "opid" : 15710430, "active" : true, "lockType" : "read", "waitingForLock" : false, "secs_running" : 17966, "op" : "query", "ns" : "app_wowsearch.events.day", "query" : { "$msg" : "query not recording (too large)" }, "client" : "127.0.0.1:33900", "desc" : "conn", "threadId" : "0x7f1e8daf5700", "connectionId" : 19963, "msg" : "m/r: (1/3) emit phase 0/835941 0%", "progress" : { "done" : 0, "total" : 835941 }, "numYields" : 3 }, db.serverStatus() { "host" : "data2:27018", "version" : "2.0.2", "process" : "mongod", "uptime" : 602434, "uptimeEstimate" : 594271, "localTime" : ISODate("2012-04-16T00:59:47.391Z"), "globalLock" : { "totalTime" : 602434261059, "lockTime" : 56141056582, "ratio" : 0.09319034492379538, "currentQueue" : { "total" : 8, "readers" : 5, "writers" : 3 }, "activeClients" : { "total" : 7, "readers" : 5, "writers" : 2 } },
Here is the map reduce code, a few utility functions are not provided. If you want them, I can provide too.
function rollup_cycle_users(cycle, firstday) { //{{{ var m = function() { //{{{ var l = this.l || {}; var mk = {id: this.id, l: {}}; var keys = ['s', 'rt', 'rc', 'rf', 'va']; keys.forEach(function(k) { mk.l[k] = l[k] || null; }); var mv = extend(copy_obj(this, ['model', 'ch', 'city', 'province', 'avn', 'resolution']), { pvs: 0, downloads: 0, searchs: 0, video_pls: 0, days: {}, }); var e2dayname = { search: 'sedays', download: 'dldays', pv: 'pvdays', video_play: 'vpdays', }; switch(this.e){ case 'download' : mv.downloads += this.c || 1; break; case 'pv' : mv.pvs += this.c || 1; break; case 'search' : mv.searchs += this.c || 1; break; case 'video_play' : mv.video_pls += this.c || 1; break; } var day_name = e2dayname[this.e]; if (day_name) { var d = 1 + (this.ts.getTime() - base_ts.getTime()) / (24 * 3600000); // start from 1 mv.days[day_name] = {}; mv.days[day_name][d] = true; emit(mk, mv); } }; //}}} var r = function(key, emits) { //{{{ var total0 = {downloads: 0, pvs: 0, video_pls:0, searchs:0, days: {}}; if (emits.length == 0) { return total0; } var total = extend(emits[0], total0); var tdays = total.days; emits.forEach(function(emit) { for (day_name in emit.days) { tdays[day_name] = extend(tdays[day_name] || {}, emit.days[day_name]); tdays[day_name] = tdays[day_name] || {}; for (d in emit.days[day_name]) { tdays[day_name][d] = true; } } total.downloads += emit.downloads; total.pvs += emit.pvs; total.video_pls += emit.video_pls; total.searchs += emit.searchs; }); return total; }; //}}} var f = function(key, total) { //{{{ function count(days) { days = days || []; return days.length; } function max_continuous_count(days) { //{{{ days = days || []; if (!days.length) return 0; var max_ccount = 0; var ccount = 0; days = days.concat([-1]); // guard for (i in days) { if (days[i] - days[i-1] != 1 || i == 0) { max_ccount = Math.max(max_ccount, ccount); ccount = 1; } else { ccount += 1; } } return max_ccount; } //}}} function max_delta(days) { deltas = delta(days); if (deltas.length == 0) return 0; return Math.max.apply(null, deltas); } function min_delta(days) { deltas = delta(days); if (deltas.length == 0) return 0; return Math.min.apply(null, deltas); } for (d in total.days) { total.days[d] = keys(total.days[d]).sort(function(a, b) {return a-b;}); } var days2al = {pvdays: 'pv', dldays: 'dl', vpdays: 'vp', sedays: 'se'}; //detail in map function 访问,下载,播放,搜索天数 var pvdays = total.days.pvdays || []; var dldays = total.days.dldays || []; aldays = {}; for (cdi in total.days) { cd = total.days[cdi]; if (!cd) continue; aldays[days2al[cdi]] = { d: count(cd), contd: max_continuous_count(cd), mindd: min_delta(cd), maxdd: max_delta(cd), }; } var min_dl_day = -1; if (dldays.length) { min_dl_day = Math.min(dldays); } dlpvdays = filter(pvdays, function(d) { return d > min_dl_day; }); dlpvd = count(dlpvdays); total = extend(total, {dlpvd: dlpvd}); total = extend(total, aldays); return total; } //}}} var cycle2delta = { week: '7d', month: '1m', }; var colname = get_scale_col_name(cycle, firstday); var mr_colname = 'mr.' + colname; var ts0 = get_china_day(firstday); var ts1 = date_add(ts0, cycle2delta[cycle]); var query = {ts: {$gte: ts0, $lt: ts1}}; // [ts0, ts1) db.events.day.mapReduce( m, r, { verbose: true, out: mr_colname, finalize: f, query: query, scope: { base_ts: ts0, copy_obj: copy_obj, filter: filter, extend: extend, delta: delta, keys: keys, }, jsMode: true, } ); db[mr_colname].find().forEach(function(doc) { var ndoc = extend(doc._id, doc.value); db[colname].insert(ndoc); }); } //}}}