Details
-
Bug
-
Resolution: Done
-
Major - P3
-
None
-
3.2.10
-
None
-
None
-
ALL
Description
1. 10 shards
2. a shard contains primary、secondary、arb
3. shard have more than 80000000 records (avgObjSize≈520byte)
this is example of my shard
192.168.0.23 is primary and crashed after some one query data
"rs3/192.168.0.23:27011,192.168.0.24:25003" : {
|
"db" : "fangjia",
|
"collections" : 1,
|
"objects" : 87950025,
|
"avgObjSize" : 518.1808235529211,
|
"dataSize" : 45574016386,
|
"storageSize" : 26404904960,
|
"numExtents" : 0,
|
"indexes" : 2,
|
"indexSize" : 5425774592,
|
"ok" : 1,
|
"$gleStats" : {
|
"lastOpTime" : Timestamp(0, 0),
|
"electionId" : ObjectId("7fffffff000000000000001d")
|
}
|
},
|
mongodb log:
2016-11-16T13:54:47.012+0800 I COMMAND [conn37] command mydata.sitelogs command: find { find: "sitelogs", filter: { _id: { $in: [ "f226b1970d7484ff44a2cf1f560018d0", "bd7423614bd775f909caab6da724d615", "ed409b4767daeb68c4a092a678e68436", "f5f9125ac1702974080ccae5fbb8f83f", "ff93daa0924c2308dec83a83c946979e", "e9e89c39ee75109301b648568381dcc0", "b318d896d42309c677876c2bf0d6b99e", "72a25cf48696ca43d759182a1a2dc8b5", "b67095e805cd4210ed3f5ebf43eebc60", "e5ca9d5c593d15bfe73d3316c83925b0", "400f760e2763e297957613a6388db417", "5a24a7a43e30e73c08d46a5c50a20098", "58689246918b6d161698e7904fcd14cd", "b0aff50b3d046f01f2b84e24288bd24f" ] } }, projection: { city: 1, region: 1, district_name: 1, rent_type: 1, name: 1, total_price: 1, area: 1, height: 1, floor: 1, fitment: 1, room: 1, hall: 1, toilet: 1, p_date: 1, url: 1 }, shardVersion: [ Timestamp 741000|1, ObjectId('57c7872d633ff0add97747a5') ] } planSummary: IXSCAN { _id: "hashed" } keysExamined:8 docsExamined:1 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:1 nreturned:1 reslen:610 locks:{ Global: { acquireCount: { r: 4 } }, Database: { acquireCount: { r: 2 } }, Collection: { acquireCount: { r: 2 } } } protocol:op_command 2496ms
|
2016-11-16T13:54:47.015+0800 I COMMAND [conn23] command mydata.sitelogs command: find { find: "sitelogs", filter: { _id: { $in: [ "ea160425ab4b4a02d50c4008d45a5c57", "20d8bac9ab28356658f8ff7bc0e3594b", "cc6528228c8e0f588e296401324da09d", "a8648a7e3726013d0a0a48d02282d4e8", "ca0bf8b43729e76ebe754b8c856bb8d1", "e97c2444524c420ab3690f489ce0f16c", "7ee8e31dbab0f714d43363298976a859", "b28db2ff0b527c5c32f3d77fd0df13c1", "f9d00ff3e74477a3a19e1531e2ca4e21", "403885ae0ede5aa7cb2669d15db0d678", "a169a35c12ba196f3566df0ef6f72aff", "b78b95db526b925bb5aaf4107384d4df", "38113162d571bac7f664b5d239f4c92c", "a3c4b21953333c0406baa5ef208c2942", "9c531c0ecba3fded59c65fa501de14dd", "c17b2744a0685900612a7bfff1e16968", "7714afc5d96248510a2dcbec8f3748ba", "3be1d83ca52eb7917ae3873c1ca10fb9", "bcf0a2aaf95f8f3ac693ae740afe8a09", "03401bcc72a39d80e4630a0a7ffd75d3", "493c21613d0d41333475bcaafa8f4b12", "5492173643be625f448e662b583159c7", "b248ab5c176510f0141a3c6ae89d145d", "86d798d0ed7ba938697b752e20edff1a", "888b3c7196d6d996af3a39f4d86fb360", "9b4a9e9f0769d4bf039238ee8f66b123", "89b0650e6b7c8b1655c5095cf7da43c0", "63342ba1f336854b0b675af5fe3ce9ed", "b94f3fbe11f1d5efaafade6de0d02e47", "03d14e660d87a175f2e60640052ffb28", "54192b0895d66d5048749e6403225236", "e321cea8254caf0722a14d60ca8ff5eb", "8a2792da8d74538cec9f5670339d3e5b", "3973eec0ad4f12e8a1848218b527b4fd", "4346cb04226a99bb9d6e993654ce6a88", "4b8dc0aa4bea733cd179348cb55e1906", "7ea06f16c3225837bce60a651865eb92" ] } }, projection: { city: 1, region: 1, district_name: 1, rent_type: 1, name: 1, total_price: 1, area: 1, height: 1, floor: 1, fitment: 1, room: 1, hall: 1, toilet: 1, p_date: 1, url: 1 }, shardVersion: [ Timestamp 741000|1, ObjectId('57c7872d633ff0add97747a5') ] } planSummary: IXSCAN { _id: "hashed" } keysExamined:22 docsExamined:6 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:1 nreturned:6 reslen:2379 locks:{ Global: { acquireCount: { r: 4 } }, Database: { acquireCount: { r: 2 } }, Collection: { acquireCount: { r: 2 } } } protocol:op_command 2698ms
|
2016-11-16T13:55:01.780+0800 I SHARDING [LockPinger] cluster 192.168.0.114:26026,192.168.0.115:26026,192.168.0.117:26026 pinged successfully at 2016-11-16T13:55:01.739+0800 by distributed lock pinger '192.168.0.114:26026,192.168.0.115:26026,192.168.0.117:26026/sha-22:27011:1479275671:1519627434', sleeping for 30000ms
|
2016-11-16T13:55:31.963+0800 I NETWORK [conn34] end connection 192.168.0.67:33766 (39 connections now open)
|
2016-11-16T13:55:31.966+0800 I NETWORK [conn35] end connection 192.168.0.67:33767 (39 connections now open)
|
2016-11-16T13:55:31.967+0800 I NETWORK [conn33] end connection 192.168.0.67:33765 (39 connections now open)
|
2016-11-16T13:55:32.089+0800 I NETWORK [conn18] end connection 192.168.0.67:33749 (39 connections now open)
|
2016-11-16T13:55:35.036+0800 I NETWORK [conn25] end connection 192.168.0.67:33756 (39 connections now open)
|
2016-11-16T13:55:35.579+0800 I NETWORK [conn38] end connection 192.168.0.67:33774 (39 connections now open)
|
2016-11-16T13:55:36.210+0800 I NETWORK [conn21] end connection 192.168.0.67:33752 (39 connections now open)
|
2016-11-16T13:55:38.124+0800 I NETWORK [conn20] end connection 192.168.0.67:33750 (39 connections now open)
|
2016-11-16T13:55:45.073+0800 I NETWORK [conn19] end connection 192.168.0.67:33748 (39 connections now open)
|
2016-11-16T13:55:45.301+0800 I NETWORK [conn29] end connection 192.168.0.67:33757 (39 connections now open)
|
2016-11-16T13:56:28.762+0800 E STORAGE [conn30] WiredTiger (5) [1479275788:762476][38660:0x7f1de3aa2700], file:mydata/index-11-7161245346684433988.wt, WT_CURSOR.search_near: /home/sitelogs/mongo_data_sitelogs_rs7_pri/mydata/index-11-7161245346684433988.wt: handle-read: pread: failed to read 12288 bytes at offset 1044275200: Input/output error
|
2016-11-16T13:56:28.765+0800 I NETWORK [conn43] end connection 192.168.0.67:33777 (40 connections now open)
|
2016-11-16T13:56:28.766+0800 I - [conn30] Invariant failure: ret resulted in status UnknownError: 5: Input/output error at src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp 831
|
2016-11-16T13:56:28.767+0800 I NETWORK [conn39] end connection 192.168.0.67:33775 (29 connections now open)
|
2016-11-16T13:56:28.767+0800 I NETWORK [conn22] end connection 192.168.0.67:33753 (29 connections now open)
|
2016-11-16T13:56:28.767+0800 I NETWORK [conn40] end connection 192.168.0.67:33776 (29 connections now open)
|
2016-11-16T13:56:28.769+0800 I NETWORK [conn23] end connection 192.168.0.67:33754 (26 connections now open)
|
2016-11-16T13:56:28.781+0800 I CONTROL [conn30]
|
0x13092a2 0x12b3eb8 0x129c21d 0x10686ed 0x106d2c0 0xbbc250 0xbafe61 0xbd4bba 0xbcb84e 0xba6652 0xe04cd2 0xe0605d 0xe0671f 0xdcd386 0xdcde4b 0xb00eee 0xb856c3 0xb86554 0xadceb0 0xc9f115 0x964c8c 0x12c3575 0x307b0079d1 0x307ace8b6d
|
----- BEGIN BACKTRACE -----
|
{"backtrace":[{"b":"400000","o":"F092A2","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"EB3EB8","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"E9C21D","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"400000","o":"C686ED"},{"b":"400000","o":"C6D2C0"},{"b":"400000","o":"7BC250","s":"_ZN5mongo9IndexScan4workEPm"},{"b":"400000","o":"7AFE61","s":"_ZN5mongo10FetchStage4workEPm"},{"b":"400000","o":"7D4BBA","s":"_ZN5mongo16ShardFilterStage4workEPm"},{"b":"400000","o":"7CB84E","s":"_ZN5mongo15ProjectionStage4workEPm"},{"b":"400000","o":"7A6652","s":"_ZN5mongo15CachedPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE"},{"b":"400000","o":"A04CD2","s":"_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE"},{"b":"400000","o":"A0605D","s":"_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EEPKNS_10CollectionERKSsNS0_11YieldPolicyE"},{"b":"400000","o":"A0671F","s":"_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EEPKNS_10CollectionENS0_11YieldPolicyE"},{"b":"400000","o":"9CD386","s":"_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS5_EENS_12PlanExecutor11YieldPolicyEm"},{"b":"400000","o":"9CDE4B","s":"_ZN5mongo15getExecutorFindEPNS_16OperationContextEPNS_10CollectionERKNS_15NamespaceStringESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS8_EENS_12PlanExecutor11YieldPolicyE"},{"b":"400000","o":"700EEE","s":"_ZN5mongo7FindCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE"},{"b":"400000","o":"7856C3","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"400000","o":"786554","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"400000","o":"6DCEB0","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"400000","o":"89F115","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"564C8C","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE"},{"b":"400000","o":"EC3575","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"307B000000","o":"79D1"},{"b":"307AC00000","o":"E8B6D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.2.10", "gitVersion" : "79d9b3ab5ce20f51c272b4411202710a082d0317", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-431.el6.x86_64", "version" : "#1 SMP Fri Nov 22 03:15:09 UTC 2013", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF47959000", "elfType" : 3 }, { "path" : "/lib64/librt.so.1", "elfType" : 3 }, { "path" : "/lib64/libdl.so.2", "elfType" : 3 }, { "path" : "/lib64/libm.so.6", "elfType" : 3 }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3 }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3 }, { "path" : "/lib64/libc.so.6", "elfType" : 3 }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
|
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x13092a2]
|
mongod(_ZN5mongo10logContextEPKc+0x138) [0x12b3eb8]
|
mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0xAD) [0x129c21d]
|
mongod(+0xC686ED) [0x10686ed]
|
mongod(+0xC6D2C0) [0x106d2c0]
|
mongod(_ZN5mongo9IndexScan4workEPm+0x290) [0xbbc250]
|
mongod(_ZN5mongo10FetchStage4workEPm+0x161) [0xbafe61]
|
mongod(_ZN5mongo16ShardFilterStage4workEPm+0x5A) [0xbd4bba]
|
mongod(_ZN5mongo15ProjectionStage4workEPm+0x4E) [0xbcb84e]
|
mongod(_ZN5mongo15CachedPlanStage12pickBestPlanEPNS_15PlanYieldPolicyE+0x192) [0xba6652]
|
mongod(_ZN5mongo12PlanExecutor12pickBestPlanENS0_11YieldPolicyE+0xB2) [0xe04cd2]
|
mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EEPKNS_10CollectionERKSsNS0_11YieldPolicyE+0x14D) [0xe0605d]
|
mongod(_ZN5mongo12PlanExecutor4makeEPNS_16OperationContextESt10unique_ptrINS_10WorkingSetESt14default_deleteIS4_EES3_INS_9PlanStageES5_IS8_EES3_INS_13QuerySolutionES5_ISB_EES3_INS_14CanonicalQueryES5_ISE_EEPKNS_10CollectionENS0_11YieldPolicyE+0xBF) [0xe0671f]
|
mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS5_EENS_12PlanExecutor11YieldPolicyEm+0x136) [0xdcd386]
|
mongod(_ZN5mongo15getExecutorFindEPNS_16OperationContextEPNS_10CollectionERKNS_15NamespaceStringESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS8_EENS_12PlanExecutor11YieldPolicyE+0x7B) [0xdcde4b]
|
mongod(_ZN5mongo7FindCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderE+0x5FE) [0xb00eee]
|
mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x473) [0xb856c3]
|
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x404) [0xb86554]
|
mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x1F0) [0xadceb0]
|
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB45) [0xc9f115]
|
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0xEC) [0x964c8c]
|
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x325) [0x12c3575]
|
libpthread.so.0(+0x79D1) [0x307b0079d1]
|
libc.so.6(clone+0x6D) [0x307ace8b6d]
|
----- END BACKTRACE -----
|
2016-11-16T13:56:28.782+0800 I - [conn30]
|
|
|
***aborting after invariant() failure
|