-
Type:
Bug
-
Resolution: Incomplete
-
Priority:
Critical - P2
-
None
-
Affects Version/s: None
-
Component/s: Sharding
-
Labels:None
-
ALL
-
We have a sharded cluster with two shards. I added a third shard following the standard procedure. I removed the third shard following standard procedure next.
Everything seemed fine with sh.status() except some of our mongos instances had the following crash/assert trying to look for that 3rd shard that I removed.
2015-04-08T20:30:05.120+0000 I - [conn127] Assertion: 13129:can't find shard for: ib3_prod_rs2
2015-04-08T20:30:05.126+0000 I CONTROL [conn127]
0xa5ac19 0xa01b71 0x9e68ef 0x9e699c 0x970de5 0x8bc56b 0x8bccc3 0x91a5c6 0x9a336e 0x9a2f67 0x9a3d5c 0x9791aa 0x64251c 0x64a563 0x64e0b9 0x980849 0x96bdfa 0x5d6df8 0xa0dd9b 0x7f2d9c265f18 0x7f2d9ad26b2d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"65AC19"},{"b":"400000","o":"601B71"},{"b":"400000","o":"5E68EF"},{"b":"400000","o":"5E699C"},{"b":"400000","o":"570DE5"},{"b":"400000","o":"4BC56B"},{"b":"400000","o":"4BCCC3"},{"b":"400000","o":"51A5C6"},{"b":"400000","o":"5A336E"},{"b":"400000","o":"5A2F67"},{"b":"400000","o":"5A3D5C"},{"b":"400000","o":"5791AA"},{"b":"400000","o":"24251C"},{"b":"400000","o":"24A563"},{"b":"400000","o":"24E0B9"},{"b":"400000","o":"580849"},{"b":"400000","o":"56BDFA"},{"b":"400000","o":"1D6DF8"},{"b":"400000","o":"60DD9B"},{"b":"7F2D9C25E000","o":"7F18"},{"b":"7F2D9AC44000","o":"E2B2D"}],"processInfo":{ "mongodbVersion" : "3.0.1", "gitVersion" : "534b5a3f9d10f00cd27737fbcd951032248b5952", "uname" : { "sysname" : "Linux", "release" : "3.14.35-28.38.amzn1.x86_64", "version" : "#1 SMP Wed Mar 11 22:50:37 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E491F456D9C74A318783ACEF59814D6F7290912B" }, { "b" : "7FFE41052000", "elfType" : 3, "buildId" : "8DD7A7F645E37D599573A937DBAA3E931F55DE40" }, { "b" : "7F2D9C25E000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "A8E8015F84374807F6920DF95D16BBAAAC02AB41" }, { "b" : "7F2D9BFF1000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "A5C6FCE56D21542B9535D5FBCFE74EBA71D24613" }, { "b" : "7F2D9BC0D000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "75565934551BECB21A027BE0A3AE28A62AEEB7EE" }, { "b" : "7F2D9BA05000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "CC3700E5EACA3A520D1FD1BA3B469F2EFB1B8F11" }, { "b" : "7F2D9B801000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "3D5EB706F159BCCB9A3B3E5F528B512E2C500F90" }, { "b" : "7F2D9B4FD000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7F2D9B1FF000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "226E54A756FDC288551092B4FDE818C8D6D6FC04" }, { "b" : "7F2D9AFE9000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7F2D9AC44000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "D40AD0538A2FFAB96E5540EF73816C675CC0AF5E" }, { "b" : "7F2D9C47A000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "37F70DE2ABEA184F24518EB5CC538FF0F54E081F" }, { "b" : "7F2D9AA01000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "72C1DB5E2447A90D1BF34065BCC031B7263FFBAC" }, { "b" : "7F2D9A71C000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "2B8787E8E0C317CF820E5D830D923BC744E497F4" }, { "b" : "7F2D9A519000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "622F315EB5CB2F791E9B64020692EBA98195D06D" }, { "b" : "7F2D9A2EE000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B10FBFEC246C4EAD1719D16090D0BE54904BBFC9" }, { "b" : "7F2D9A0D8000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "87B4EBF2183C8EA4AB657212203EFFE6340E2F4F" }, { "b" : "7F2D99ECD000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "7292C0673D7C116E3389D3FFA67087A6B9287A71" }, { "b" : "7F2D99CCA000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7F2D99AB3000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F01971089728188EFA4AA473D8B2DE22CA6DE963" }, { "b" : "7F2D99892000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "803D7EF21A989677D056E52BAEB9AB5B154FB9D9" } ] }}
mongos(_ZN5mongo15printStackTraceERSo+0x29) [0xa5ac19]
mongos(_ZN5mongo10logContextEPKc+0xE1) [0xa01b71]
mongos(_ZN5mongo11msgassertedEiPKc+0xAF) [0x9e68ef]
mongos(+0x5E699C) [0x9e699c]
mongos(_ZN5mongo5Shard5resetERKSs+0x1F5) [0x970de5]
mongos(_ZN5mongo12ChunkManager5_loadERKSsRSt3mapINS_7BSONObjEN5boost10shared_ptrIKNS_5ChunkEEENS_10BSONObjCmpESaISt4pairIKS4_S9_EEERSt3setINS_5ShardESt4lessISI_ESaISI_EERS3_ISsNS_12ChunkVersionESJ_ISsESaISB_IS1_SO_EEEPKS0_+0x60B) [0x8bc56b]
mongos(_ZN5mongo12ChunkManager18loadExistingRangesERKSsPKS0_+0x163) [0x8bccc3]
mongos(_ZN5mongo8DBConfig15getChunkManagerERKSsbb+0x696) [0x91a5c6]
mongos(_ZN5mongo17checkShardVersionEPNS_12DBClientBaseERKSsN5boost10shared_ptrIKNS_12ChunkManagerEEEbi+0x1B1E) [0x9a336e]
mongos(_ZN5mongo17checkShardVersionEPNS_12DBClientBaseERKSsN5boost10shared_ptrIKNS_12ChunkManagerEEEbi+0x1717) [0x9a2f67]
mongos(_ZN5mongo14VersionManager19checkShardVersionCBEPNS_15ShardConnectionEbi+0x5C) [0x9a3d5c]
mongos(_ZN5mongo15ShardConnection11_finishInitEv+0xEA) [0x9791aa]
mongos(_ZN5mongo27ParallelSortClusteredCursor28setupVersionAndHandleSlaveOkEN5boost10shared_ptrINS_23ParallelConnectionStateEEERKNS_5ShardENS2_IS5_EERKNS_15NamespaceStringERKSsNS2_IKNS_12ChunkManagerEEE+0x11C) [0x64251c]
mongos(_ZN5mongo27ParallelSortClusteredCursor9startInitEv+0xD63) [0x64a563]
mongos(_ZN5mongo27ParallelSortClusteredCursor8fullInitEv+0x9) [0x64e0b9]
mongos(_ZN5mongo8Strategy7queryOpERNS_7RequestE+0x3B9) [0x980849]
mongos(_ZN5mongo7Request7processEi+0x53A) [0x96bdfa]
mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x58) [0x5d6df8]
mongos(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xa0dd9b]
libpthread.so.0(+0x7F18) [0x7f2d9c265f18]
libc.so.6(clone+0x6D) [0x7f2d9ad26b2d]
----- END BACKTRACE -----