mongodb 2.0.0 secondary keeps throwing exceptions until it became stale. old versions (1.8.x) don't have this bug.

Fri Oct 14 02:54:02 [initandlisten] connection accepted from 192.168.168.252:53261 #847
Fri Oct 14 02:54:04 [initandlisten] connection accepted from 192.168.168.252:53644 #848
Fri Oct 14 02:54:11 [rsSync] jiepang_production.user Assertion failure 0 db/ops/update.cpp 691
0x57d716 0x5885cb 0x95cac8 0x95c540 0x941214 0x941cc7 0x945e25 0x948375 0x82c984 0x81d9db 0x81ed7c 0x820108 0x820158 0x8205d0 0xa9ea30 0x7ff38ffe19ca 0x7ff38f59070d
/usr/bin/mongod(_ZN5mongo12sayDbContextEPKc+0x96) [0x57d716]
/usr/bin/mongod(_ZN5mongo8assertedEPKcS1_j+0xfb) [0x5885cb]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0x10c8) [0x95cac8]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0xb40) [0x95c540]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsEv+0x64) [0x941214]
/usr/bin/mongod() [0x941cc7]
/usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugEPNS_11RemoveSaverE+0x2b75) [0x945e25]
/usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugE+0x125) [0x948375]
/usr/bin/mongod(_ZN5mongo21applyOperation_inlockERKNS_7BSONObjEb+0x5e4) [0x82c984]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl9syncApplyERKNS_7BSONObjE+0x1fb) [0x81d9db]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl8syncTailEv+0xe3c) [0x81ed7c]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl11_syncThreadEv+0xc8) [0x820108]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl10syncThreadEv+0x48) [0x820158]
/usr/bin/mongod(_ZN5mongo15startSyncThreadEv+0xa0) [0x8205d0]
/usr/bin/mongod(thread_proxy+0x80) [0xa9ea30]
/lib/libpthread.so.0(+0x69ca) [0x7ff38ffe19ca]
/lib/libc.so.6(clone+0x6d) [0x7ff38f59070d]
Fri Oct 14 02:54:11 [rsSync] replSet syncTail: 0 assertion db/ops/update.cpp:691, syncing: { ts: Timestamp 1318532051000|7, h: -906611776605637944, op: "u", ns: "jiepang_production.user", o2:

{ _id: 642243809 }

, o: { $set:

{ citystats.喀什.lcd: "2011-05-24", citystats.天津.lcd: "2010-10-04", citystats.泰安.lcd: "2011-08-13", citystats.杭州.ncl: 13, citystats.阿克苏.ncl: 2, citystats.上海.lcd: "2011-09-10", citystats.杭州.ncd: 7, citystats.阿克苏.ncd: 2, citystats.博尔塔拉州.ncl: 1, citystats.北京.ncl: 98, citystats.阿克苏地区.ncd: 1, citystats.泰安.ncl: 3, citystats.北京.ncd: 90, citystats.阿克苏地区.ncl: 1, citystats.上海.ncd: 1, citystats.阿克苏.lcd: "2011-05-19", citystats.海外.ncd: 1, citystats.天津.ncd: 1, citystats.上海.ncl: 1, citystats.海外.ncl: 1, citystats.天津.ncl: 1, citystats.和田.ncd: 1, citystats.深圳.ncl: 2, citystats.博尔塔拉州.ncd: 1, citystats.和田.ncl: 1, citystats.青岛.lcd: "2011-06-06", citystats.深圳.lcd: "2011-01-25", citystats.深圳.ncd: 2, citystats.阿克苏地区.lcd: "2011-05-16", citystats.海外.lcd: "2011-08-13", citystats.青岛.ncd: 3, citystats.北京.lcd: "2011-10-12", citystats.秦皇岛.lcd: "2011-07-16", citystats.和田.lcd: "2011-05-26", citystats.青岛.ncl: 4, citystats.伊宁.ncl: 1, citystats.喀什.ncl: 1, citystats.乌鲁木齐.lcd: "2011-06-03", citystats.伊宁.ncd: 1, citystats.博尔塔拉州.lcd: "2011-06-01", citystats.喀什.ncd: 1, citystats.秦皇岛.ncd: 2, citystats.泰安.ncd: 1, citystats.伊宁.lcd: "2011-05-27", citystats.乌鲁木齐.ncl: 5, citystats.乌鲁木齐.ncd: 4, citystats.秦皇岛.ncl: 3, citystats.杭州.lcd: "2011-08-07" }

} }
Fri Oct 14 02:54:32 [conn838] end connection 192.168.168.252:55504
Fri Oct 14 02:54:32 [conn837] end connection 192.168.168.252:50230
Fri Oct 14 02:54:42 [rsSync] replSet syncing to: 192.168.168.204:27017
Fri Oct 14 02:54:42 [rsSync] jiepang_production.user Assertion failure 0 db/ops/update.cpp 691
0x57d716 0x5885cb 0x95cac8 0x95c540 0x941214 0x941cc7 0x945e25 0x948375 0x82c984 0x81d9db 0x81ed7c 0x820108 0x820158 0x8205d0 0xa9ea30 0x7ff38ffe19ca 0x7ff38f59070d
/usr/bin/mongod(_ZN5mongo12sayDbContextEPKc+0x96) [0x57d716]
/usr/bin/mongod(_ZN5mongo8assertedEPKcS1_j+0xfb) [0x5885cb]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0x10c8) [0x95cac8]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0xb40) [0x95c540]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsEv+0x64) [0x941214]
/usr/bin/mongod() [0x941cc7]
/usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugEPNS_11RemoveSaverE+0x2b75) [0x945e25]
/usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugE+0x125) [0x948375]
/usr/bin/mongod(_ZN5mongo21applyOperation_inlockERKNS_7BSONObjEb+0x5e4) [0x82c984]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl9syncApplyERKNS_7BSONObjE+0x1fb) [0x81d9db]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl8syncTailEv+0xe3c) [0x81ed7c]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl11_syncThreadEv+0xc8) [0x820108]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl10syncThreadEv+0x48) [0x820158]
/usr/bin/mongod(_ZN5mongo15startSyncThreadEv+0xa0) [0x8205d0]
/usr/bin/mongod(thread_proxy+0x80) [0xa9ea30]
/lib/libpthread.so.0(+0x69ca) [0x7ff38ffe19ca]
/lib/libc.so.6(clone+0x6d) [0x7ff38f59070d]
Fri Oct 14 02:54:42 [rsSync] replSet syncTail: 0 assertion db/ops/update.cpp:691, syncing: { ts: Timestamp 1318532051000|7, h: -906611776605637944, op: "u", ns: "jiepang_production.user", o2:

{ _id: 642243809 }

, o: { $set:

{ citystats.喀什.lcd: "2011-05-24", citystats.天津.lcd: "2010-10-04", citystats.泰安.lcd: "2011-08-13", citystats.杭州.ncl: 13, citystats.阿克苏.ncl: 2, citystats.上海.lcd: "2011-09-10", citystats.杭州.ncd: 7, citystats.阿克苏.ncd: 2, citystats.博尔塔拉州.ncl: 1, citystats.北京.ncl: 98, citystats.阿克苏地区.ncd: 1, citystats.泰安.ncl: 3, citystats.北京.ncd: 90, citystats.阿克苏地区.ncl: 1, citystats.上海.ncd: 1, citystats.阿克苏.lcd: "2011-05-19", citystats.海外.ncd: 1, citystats.天津.ncd: 1, citystats.上海.ncl: 1, citystats.海外.ncl: 1, citystats.天津.ncl: 1, citystats.和田.ncd: 1, citystats.深圳.ncl: 2, citystats.博尔塔拉州.ncd: 1, citystats.和田.ncl: 1, citystats.青岛.lcd: "2011-06-06", citystats.深圳.lcd: "2011-01-25", citystats.深圳.ncd: 2, citystats.阿克苏地区.lcd: "2011-05-16", citystats.海外.lcd: "2011-08-13", citystats.青岛.ncd: 3, citystats.北京.lcd: "2011-10-12", citystats.秦皇岛.lcd: "2011-07-16", citystats.和田.lcd: "2011-05-26", citystats.青岛.ncl: 4, citystats.伊宁.ncl: 1, citystats.喀什.ncl: 1, citystats.乌鲁木齐.lcd: "2011-06-03", citystats.伊宁.ncd: 1, citystats.博尔塔拉州.lcd: "2011-06-01", citystats.喀什.ncd: 1, citystats.秦皇岛.ncd: 2, citystats.泰安.ncd: 1, citystats.伊宁.lcd: "2011-05-27", citystats.乌鲁木齐.ncl: 5, citystats.乌鲁木齐.ncd: 4, citystats.秦皇岛.ncl: 3, citystats.杭州.lcd: "2011-08-07" }

} }
Fri Oct 14 02:54:44 [initandlisten] connection accepted from 192.168.168.252:60769 #849
Fri Oct 14 02:55:01 [initandlisten] connection accepted from 192.168.168.252:35721 #850
Fri Oct 14 02:55:01 [conn850] end connection 192.168.168.252:35721
Fri Oct 14 02:55:11 [initandlisten] connection accepted from 192.168.168.252:37529 #851
Fri Oct 14 02:55:13 [rsSync] replSet syncing to: 192.168.168.204:27017
Fri Oct 14 02:55:13 [rsSync] jiepang_production.user Assertion failure 0 db/ops/update.cpp 691
0x57d716 0x5885cb 0x95cac8 0x95c540 0x941214 0x941cc7 0x945e25 0x948375 0x82c984 0x81d9db 0x81ed7c 0x820108 0x820158 0x8205d0 0xa9ea30 0x7ff38ffe19ca 0x7ff38f59070d
/usr/bin/mongod(_ZN5mongo12sayDbContextEPKc+0x96) [0x57d716]
/usr/bin/mongod(_ZN5mongo8assertedEPKcS1_j+0xfb) [0x5885cb]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0x10c8) [0x95cac8]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsINS_14BSONObjBuilderEEEvRKSsRT_RKNS_7BSONObjE+0xb40) [0x95c540]
/usr/bin/mongod(_ZN5mongo11ModSetState17createNewFromModsEv+0x64) [0x941214]
/usr/bin/mongod() [0x941cc7]
/usr/bin/mongod(_ZN5mongo14_updateObjectsEbPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugEPNS_11RemoveSaverE+0x2b75) [0x945e25]
/usr/bin/mongod(_ZN5mongo13updateObjectsEPKcRKNS_7BSONObjES2_bbbRNS_7OpDebugE+0x125) [0x948375]
/usr/bin/mongod(_ZN5mongo21applyOperation_inlockERKNS_7BSONObjEb+0x5e4) [0x82c984]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl9syncApplyERKNS_7BSONObjE+0x1fb) [0x81d9db]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl8syncTailEv+0xe3c) [0x81ed7c]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl11_syncThreadEv+0xc8) [0x820108]
/usr/bin/mongod(_ZN5mongo11ReplSetImpl10syncThreadEv+0x48) [0x820158]
/usr/bin/mongod(_ZN5mongo15startSyncThreadEv+0xa0) [0x8205d0]
/usr/bin/mongod(thread_proxy+0x80) [0xa9ea30]
/lib/libpthread.so.0(+0x69ca) [0x7ff38ffe19ca]
/lib/libc.so.6(clone+0x6d) [0x7ff38f59070d]
Fri Oct 14 02:55:13 [rsSync] replSet syncTail: 0 assertion db/ops/update.cpp:691, syncing: { ts: Timestamp 1318532051000|7, h: -906611776605637944, op: "u", ns: "jiepang_production.user", o2:

{ _id: 642243809 }

, o: { $set:

{ citystats.喀什.lcd: "2011-05-24", citystats.天津.lcd: "2010-10-04", citystats.泰安.lcd: "2011-08-13", citystats.杭州.ncl: 13, citystats.阿克苏.ncl: 2, citystats.上海.lcd: "2011-09-10", citystats.杭州.ncd: 7, citystats.阿克苏.ncd: 2, citystats.博尔塔拉州.ncl: 1, citystats.北京.ncl: 98, citystats.阿克苏地区.ncd: 1, citystats.泰安.ncl: 3, citystats.北京.ncd: 90, citystats.阿克苏地区.ncl: 1, citystats.上海.ncd: 1, citystats.阿克苏.lcd: "2011-05-19", citystats.海外.ncd: 1, citystats.天津.ncd: 1, citystats.上海.ncl: 1, citystats.海外.ncl: 1, citystats.天津.ncl: 1, citystats.和田.ncd: 1, citystats.深圳.ncl: 2, citystats.博尔塔拉州.ncd: 1, citystats.和田.ncl: 1, citystats.青岛.lcd: "2011-06-06", citystats.深圳.lcd: "2011-01-25", citystats.深圳.ncd: 2, citystats.阿克苏地区.lcd: "2011-05-16", citystats.海外.lcd: "2011-08-13", citystats.青岛.ncd: 3, citystats.北京.lcd: "2011-10-12", citystats.秦皇岛.lcd: "2011-07-16", citystats.和田.lcd: "2011-05-26", citystats.青岛.ncl: 4, citystats.伊宁.ncl: 1, citystats.喀什.ncl: 1, citystats.乌鲁木齐.lcd: "2011-06-03", citystats.伊宁.ncd: 1, citystats.博尔塔拉州.lcd: "2011-06-01", citystats.喀什.ncd: 1, citystats.秦皇岛.ncd: 2, citystats.泰安.ncd: 1, citystats.伊宁.lcd: "2011-05-27", citystats.乌鲁木齐.ncl: 5, citystats.乌鲁木齐.ncd: 4, citystats.秦皇岛.ncl: 3, citystats.杭州.lcd: "2011-08-07" }

} }
Fri Oct 14 02:55:25 [conn848] end connection 192.168.168.252:53644
Fri Oct 14 02:55:25 [conn839] end connection 192.168.168.252:58788
Fri Oct 14 02:55:37 [initandlisten] connection accepted from 192.168.168.252:42163 #852
Fri Oct 14 02:55:42 [clientcursormon] mem (MB) res:56779 virt:174122 mapped:86711
Fri Oct 14 02:55:44 [rsSync] replSet syncing to: 192.168.168.204:27017
...