153288335-2015-10-21T09:52:40.980+0000 I QUERY [conn190654] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153288819-2015-10-21T09:52:40.980+0000 I QUERY [conn189833] ntoskip:0 ntoreturn:1 153288895-2015-10-21T09:52:40.980+0000 I QUERY [conn191463] assertion 13388 [Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3327 ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ns:Appsee.$cmd query:{ findAndModify: "HeatmapReports", new: true, upsert: true, update: { $push: { c.9: { $each: [ [ 140, 40 ], [ 151, 49 ] ], $slice: 100 } } }, query: { d: new Date(1445385600000), g: 0, o: 0, as: ObjectId('561ca8fcbf86ce4ffd4c4ea9'), r: false, ri: ObjectId('561cac056ff5a34f2c2c83a0') } } 153289677-2015-10-21T09:52:40.980+0000 I QUERY [conn191463] ntoskip:0 ntoreturn:1 153289753-2015-10-21T09:52:40.980+0000 I QUERY [conn189833] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153290237-2015-10-21T09:52:40.980+0000 I QUERY [conn191463] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153290721-2015-10-21T09:52:40.980+0000 I QUERY [conn189100] assertion 13388 [Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3327 ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ns:Appsee.$cmd query:{ findAndModify: "HeatmapReports", new: true, upsert: true, update: { $push: { c.9: { $each: [ [ 218, 576 ], [ 218, 576 ], [ 215, 530 ], [ 208, 480 ] ], $slice: 100 } } }, query: { d: new Date(1445385600000), o: 1, as: ObjectId('55a3819b859aed8d54622573'), ri: ObjectId('56150893723ac73a0ef02890') } } 153291517-2015-10-21T09:52:40.981+0000 I QUERY [conn189100] ntoskip:0 ntoreturn:1 153291593-2015-10-21T09:52:40.981+0000 I QUERY [conn189100] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153292077-2015-10-21T09:52:40.981+0000 I QUERY [conn191466] assertion 13388 [Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3327 ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ns:Appsee.$cmd query:{ findAndModify: "HeatmapReports", new: true, upsert: true, update: { $push: { c.9: { $each: [ [ 953, 1821 ], [ 951, 1821 ] ], $slice: 100 } } }, query: { d: new Date(1445385600000), g: 0, o: 0, as: ObjectId('560ca5872f1713a9b197172d'), r: true, ri: ObjectId('56166a7b9a422b4392341e8f') } } 153292862-2015-10-21T09:52:40.981+0000 I QUERY [conn191466] ntoskip:0 ntoreturn:1 153292938-2015-10-21T09:52:40.981+0000 I QUERY [conn190648] assertion 13388 [Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3327 ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ns:Appsee.$cmd query:{ findAndModify: "HeatmapReports", new: true, upsert: true, update: { $push: { c.8: { $each: [ [ 9, 243 ], [ 10, 253 ], [ 18, 274 ], [ 18, 277 ], [ 26, 293 ] ], $slice: 100 } } }, query: { d: new Date(1445385600000), g: 5, o: 0, as: ObjectId('55fd68512f1713a9b1834168'), r: false, ri: ObjectId('55fe4c0ed1d1d50e5ea1cca4') } } 153293758-2015-10-21T09:52:40.981+0000 I QUERY [conn191466] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153294242-2015-10-21T09:52:40.981+0000 I NETWORK [conn190654] end connection 10.40.7.177:57660 (924 connections now open) 153294355-2015-10-21T09:52:40.981+0000 I QUERY [conn190648] ntoskip:0 ntoreturn:1 153294431-2015-10-21T09:52:40.981+0000 I QUERY [conn190648] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153294915-2015-10-21T09:52:40.981+0000 I NETWORK [conn189833] end connection 10.40.7.177:56663 (923 connections now open) 153295028-2015-10-21T09:52:40.981+0000 I NETWORK [conn191463] end connection 10.40.7.177:58495 (923 connections now open) 153295141-2015-10-21T09:52:40.981+0000 I SHARDING [conn191409] waiting till out of critical section 153295231-2015-10-21T09:52:40.981+0000 I NETWORK [conn189100] end connection 10.40.7.177:56144 (921 connections now open) 153295344-2015-10-21T09:52:40.981+0000 I SHARDING [conn189857] waiting till out of critical section 153295434-2015-10-21T09:52:40.981+0000 I SHARDING [conn193492] waiting till out of critical section 153295524-2015-10-21T09:52:40.981+0000 I NETWORK [conn191466] end connection 10.37.137.232:58460 (920 connections now open) 153295639-2015-10-21T09:52:40.981+0000 I NETWORK [conn190648] end connection 10.37.137.232:57696 (920 connections now open) 153295754-2015-10-21T09:52:40.981+0000 I SHARDING [conn191422] waiting till out of critical section 153295844-2015-10-21T09:52:40.981+0000 I SHARDING [conn190679] waiting till out of critical section 153295934-2015-10-21T09:52:40.982+0000 I - [conn195100] Assertion: 10334:BSONObj size: 1127114085 (0x432E6565) is invalid. Size must be between 0 and 16793600(16MB) First element: stomEventPropertyReports: ?type=117 153296147-2015-10-21T09:52:40.987+0000 I SHARDING [conn63619] moveChunk migrate commit accepted by TO-shard: { active: false, ns: "Appsee.HeatmapReports", from: "rsAppsee2/10.35.149.150:27017,in.db2m1.appsee.com:27017,in.db2m2.appsee.com:27017", min: { as: ObjectId('53bc4dd571f6466d7a3c22dc'), ri: ObjectId('53c9233c761bd045e985120c'), o: 1 }, max: { as: ObjectId('53bc60f671f6466d7a3c6f5d'), ri: ObjectId('53c9b55f761bd045e989d71a'), o: 2 }, shardKeyPattern: { as: 1.0, ri: 1.0, o: 1.0 }, state: "done", counts: { cloned: 17861, clonedBytes: 8218132, catchup: 0, steady: 0 }, ok: 1.0, $gleStats: { lastOpTime: Timestamp 0|0, electionId: ObjectId('5614daebaf3f6d27f2e2fdec') } } 153296817-2015-10-21T09:52:40.988+0000 I SHARDING [conn63619] moveChunk updating self version to: 3328|1||54be86fcf5d6feabbc856443 through { as: ObjectId('53bc60f671f6466d7a3c6f5d'), ri: ObjectId('53c9b55f761bd045e989d71a'), o: 2 } -> { as: ObjectId('53bd06a371f6466d7a4348d4'), ri: ObjectId('53bd06ff761bd03722590de0'), o: 0 } for collection 'Appsee.HeatmapReports' 153297174-2015-10-21T09:52:40.991+0000 I SHARDING [conn193229] waiting till out of critical section 153297264-2015-10-21T09:52:40.992+0000 I QUERY [conn190639] assertion 13388 [Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3327 ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ( ns : Appsee.HeatmapReports, received : 3327|1||54be86fcf5d6feabbc856443, wanted : 3328|0||54be86fcf5d6feabbc856443, send ) ns:Appsee.$cmd query:{ findAndModify: "HeatmapReports", new: true, upsert: true, update: { $push: { c.9: { $each: [ [ 953, 1821 ], [ 951, 1821 ] ], $slice: 100 } } }, query: { d: new Date(1445385600000), g: 0, o: 0, as: ObjectId('560ca5872f1713a9b197172d'), r: true, ri: ObjectId('56166a7b9a422b4392341e8f') } } 153298049-2015-10-21T09:52:40.992+0000 I QUERY [conn190639] ntoskip:0 ntoreturn:1 153298125-2015-10-21T09:52:40.992+0000 I QUERY [conn190639] stale version detected during query over Appsee.$cmd : { $err: "[Appsee.HeatmapReports] shard version not ok: version mismatch detected for Appsee.HeatmapReports, stored major version 3328 does not match received 3...", code: 13388, ns: "Appsee.HeatmapReports", vReceived: Timestamp 3327000|1, vReceivedEpoch: ObjectId('54be86fcf5d6feabbc856443'), vWanted: Timestamp 3328000|0, vWantedEpoch: ObjectId('54be86fcf5d6feabbc856443') } 153298609-2015-10-21T09:52:40.993+0000 I NETWORK [conn190639] end connection 10.37.137.232:57687 (918 connections now open) 153298724-2015-10-21T09:52:40.993+0000 I SHARDING [conn191439] waiting till out of critical section 153298814-2015-10-21T09:52:41.001+0000 I SHARDING [conn190651] waiting till out of critical section 153298904-2015-10-21T09:52:41.001+0000 I SHARDING [conn193485] waiting till out of critical section 153298994-2015-10-21T09:52:41.009+0000 I SHARDING [conn193495] waiting till out of critical section 153299084-2015-10-21T09:52:41.037+0000 I CONTROL [conn195100] 153299138- 0xf6b4a9 0xf0bca1 0xef0eff 0xef0fac 0x84feff 0x92de38 0xd621b8 0xd5b0f6 0x92e8dc 0xc550a6 0xc537b1 0xa567fc 0xa56fb5 0xbed174 0xbed524 0xbedb5d 0x9ce03d 0x9cf82c 0x9cfd6b 0x9d270d 0x9f5e34 0x9f6dbd 0x9f7acb 0xbbb836 0xad2190 0x829cad 0xf1f09b 0x7f5802a8bdf3 0x7f580153f1bd 153299412:----- BEGIN BACKTRACE ----- 153299440-{"backtrace":[{"b":"400000","o":"B6B4A9"},{"b":"400000","o":"B0BCA1"},{"b":"400000","o":"AF0EFF"},{"b":"400000","o":"AF0FAC"},{"b":"400000","o":"44FEFF"},{"b":"400000","o":"52DE38"},{"b":"400000","o":"9621B8"},{"b":"400000","o":"95B0F6"},{"b":"400000","o":"52E8DC"},{"b":"400000","o":"8550A6"},{"b":"400000","o":"8537B1"},{"b":"400000","o":"6567FC"},{"b":"400000","o":"656FB5"},{"b":"400000","o":"7ED174"},{"b":"400000","o":"7ED524"},{"b":"400000","o":"7EDB5D"},{"b":"400000","o":"5CE03D"},{"b":"400000","o":"5CF82C"},{"b":"400000","o":"5CFD6B"},{"b":"400000","o":"5D270D"},{"b":"400000","o":"5F5E34"},{"b":"400000","o":"5F6DBD"},{"b":"400000","o":"5F7ACB"},{"b":"400000","o":"7BB836"},{"b":"400000","o":"6D2190"},{"b":"400000","o":"429CAD"},{"b":"400000","o":"B1F09B"},{"b":"7F5802A84000","o":"7DF3"},{"b":"7F5801449000","o":"F61BD"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "3.14.44-32.39.amzn1.x86_64", "version" : "#1 SMP Thu Jun 11 20:33:38 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "810BAA4B0BDE1EBB076BF6A544BF2724A13FAE84" }, { "b" : "7FFD23EC3000", "elfType" : 3, "buildId" : "3C9325999495818299EC622C63D9330D234E02C9" }, { "b" : "7F5802A84000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "D48D3E6672A77B603B402F661BABF75E90AD570B" }, { "b" : "7F5802817000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7F5802432000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7F580222A000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "E81013CBFA409053D58A65A0653271AB665A4619" }, { "b" : "7F5802026000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "62A8842157C62F95C3069CBF779AFCC26577A99A" }, { "b" : "7F5801D22000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7F5801A20000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5F97F8F8E5024E29717CF35998681F84D4A22D45" }, { "b" : "7F580180A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7F5801449000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "DF6DA145A649EA093507A635AF383F608E7CE3F2" }, { "b" : "7F5802CA0000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "6F90843B9087FE91955FEB0355EB0858EF9E97B2" }, { "b" : "7F5801206000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DE5A9F7A11A0881CB64E375F4DDCA58028F0FAF8" }, { "b" : "7F5800F21000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "A3E43FC66908AC8B00773707FECA3B1677AFF311" }, { "b" : "7F5800D1E000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "622F315EB5CB2F791E9B64020692EBA98195D06D" }, { "b" : "7F5800AF3000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B10FBFEC246C4EAD1719D16090D0BE54904BBFC9" }, { "b" : "7F58008DD000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "87B4EBF2183C8EA4AB657212203EFFE6340E2F4F" }, { "b" : "7F58006D2000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "7292C0673D7C116E3389D3FFA67087A6B9287A71" }, { "b" : "7F58004CF000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7F58002B5000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "6A7DA1CED90F65F27CB7B5BACDBB1C386C05F592" }, { "b" : "7F5800094000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "F5054DC94443326819FBF3065CFDF5E4726F57EE" } ] }} 153303234- mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf6b4a9] 153303290- mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf0bca1] 153303341- mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xef0eff] 153303394- mongod(+0xAF0FAC) [0xef0fac] 153303424- mongod(_ZNK5mongo7BSONObj14_assertInvalidEv+0x3DF) [0x84feff] 153303487- mongod(_ZN5mongo10Collection19aboutToDeleteCappedEPNS_16OperationContextERKNS_8RecordIdENS_10RecordDataE+0xB8) [0x92de38] 153303610- mongod(_ZN5mongo19CappedRecordStoreV111allocRecordEPNS_16OperationContextEib+0x518) [0xd621b8] 153303706- mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKNS_9DocWriterEb+0x76) [0xd5b0f6] 153303815- mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextEPKNS_9DocWriterEb+0x5C) [0x92e8dc] 153303919- mongod(+0x8550A6) [0xc550a6] 153303949- mongod(_ZN5mongo4repl5logOpEPNS_16OperationContextEPKcS4_RKNS_7BSONObjEPS5_Pbb+0x161) [0xc537b1] 153304047- mongod(_ZN5mongo11UpdateStage18transformAndUpdateERKNS_11SnapshottedINS_7BSONObjEEERNS_8RecordIdE+0x6AC) [0xa567fc] 153304164- mongod(_ZN5mongo11UpdateStage4workEPm+0x3E5) [0xa56fb5] 153304221- mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0xA4) [0xbed174] 153304337- mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x34) [0xbed524] 153304422- mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbedb5d] 153304486- mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x71D) [0x9ce03d] 153304612- mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x23C) [0x9cf82c] 153304819- mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x37B) [0x9cfd6b] 153304948- mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x15D) [0x9d270d] 153305064- mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9f5e34] 153305193- mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9f6dbd] 153305317- mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9f7acb] 153305472- mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x746) [0xbbb836] 153305610- mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xad2190] 153305737- mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829cad] 153305855- mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf1f09b] 153305932- libpthread.so.0(+0x7DF3) [0x7f5802a8bdf3] 153305975- libc.so.6(clone+0x6D) [0x7f580153f1bd] 153306015------ END BACKTRACE -----