Major - P3 Over the last few days the python driver tests have been failing against master-nightly in Jenkins when run against a sharded cluster. It's not clear what test is tickling the bug, and seems to vary from test run to test run. I'm attaching the mongos log from one run. The backtrace is:
...
2015-06-17T00:18:43.068+0000 D SHARDING [conn46] found 0 collections left and 0 collections dropped for database pymongo_test_mike
2015-06-17T00:18:43.390+0000 D NETWORK [conn46] polling for status of connection to 127.0.0.1:27016, no events
2015-06-17T00:18:43.390+0000 D NETWORK [conn46] polling for status of connection to 127.0.0.1:27021, no events
2015-06-17T00:18:43.390+0000 D NETWORK [conn46] polling for status of connection to 127.0.0.1:27022, no events
2015-06-17T00:18:43.390+0000 D SHARDING [conn46] found 1 shards listed on config server(s)
2015-06-17T00:18:43.391+0000 F - [conn46] Invalid access at address: 0
2015-06-17T00:18:43.393+0000 F - [conn46] Got signal: 11 (Segmentation fault).
0xa98bb6 0xa982d2 0xa9860e 0x7f18029795b0 0x97122b 0x971d1b 0x99b237 0x9deff8 0x9dfbbc 0x9e57c8 0x9de624 0x600ace 0xa518c1 0x7f1802971f18 0x7f18026a7b2d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"698BB6"},{"b":"400000","o":"6982D2"},{"b":"400000","o":"69860E"},{"b":"7F180296A000","o":"F5B0"},{"b":"400000","o":"57122B"},{"b":"400000","o":"571D1B"},{"b":"400000","o":"59B237"},{"b":"400000","o":"5DEFF8"},{"b":"400000","o":"5DFBBC"},{"b":"400000","o":"5E57C8"},{"b":"400000","o":"5DE624"},{"b":"400000","o":"200ACE"},{"b":"400000","o":"6518C1"},{"b":"7F180296A000","o":"7F18"},{"b":"7F18025C5000","o":"E2B2D"}],"processInfo":{ "mongodbVersion" : "3.1.5-pre-", "gitVersion" : "be88e42ef819aeb76b8625e646adf67626844c25", "compiledModules" : [ "enterprise" ], "uname" : { "sysname" : "Linux", "release" : "3.14.35-28.38.amzn1.x86_64", "version" : "#1 SMP Wed Mar 11 22:50:37 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "E68A888A295E8469F953C4C275B4EEC790FB1EB3" }, { "b" : "7FFE7669A000", "elfType" : 3, "buildId" : "8DD7A7F645E37D599573A937DBAA3E931F55DE40" }, { "b" : "7F180403E000", "path" : "/usr/lib64/libsasl2.so.2", "elfType" : 3, "buildId" : "354560FFC93703E5A80EEC8C66DF9E59DA335001" }, { "b" : "7F1803DFB000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "72C1DB5E2447A90D1BF34065BCC031B7263FFBAC" }, { "b" : "7F1803B8E000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "A5C6FCE56D21542B9535D5FBCFE74EBA71D24613" }, { "b" : "7F18037AA000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "75565934551BECB21A027BE0A3AE28A62AEEB7EE" }, { "b" : "7F18035A2000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "CC3700E5EACA3A520D1FD1BA3B469F2EFB1B8F11" }, { "b" : "7F180339E000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "3D5EB706F159BCCB9A3B3E5F528B512E2C500F90" }, { "b" : "7F180309A000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "DD6383EEAC49E9BAA9E3D1080AE932F42CF8A385" }, { "b" : "7F1802D9C000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "226E54A756FDC288551092B4FDE818C8D6D6FC04" }, { "b" : "7F1802B86000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "C52958E393BDF8E8D090F36DE0F4E620D8736FBF" }, { "b" : "7F180296A000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "A8E8015F84374807F6920DF95D16BBAAAC02AB41" }, { "b" : "7F18025C5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "D40AD0538A2FFAB96E5540EF73816C675CC0AF5E" }, { "b" : "7F1804259000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "37F70DE2ABEA184F24518EB5CC538FF0F54E081F" }, { "b" : "7F18023AE000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F01971089728188EFA4AA473D8B2DE22CA6DE963" }, { "b" : "7F1802178000", "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "DD436290CC2AC5B442DFCEB885AA43C77D2C4CC7" }, { "b" : "7F1801E93000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "2B8787E8E0C317CF820E5D830D923BC744E497F4" }, { "b" : "7F1801C68000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B10FBFEC246C4EAD1719D16090D0BE54904BBFC9" }, { "b" : "7F1801A65000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "622F315EB5CB2F791E9B64020692EBA98195D06D" }, { "b" : "7F180185A000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "7292C0673D7C116E3389D3FFA67087A6B9287A71" }, { "b" : "7F1801657000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7F1801441000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "87B4EBF2183C8EA4AB657212203EFFE6340E2F4F" }, { "b" : "7F18011C6000", "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "3A7566B3366A1989E66B5AD54CEAF58C57E2ED3A" }, { "b" : "7F1800FA5000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "803D7EF21A989677D056E52BAEB9AB5B154FB9D9" }, { "b" : "7F17FA878000", "path" : "/usr/lib64/sasl2/libsasldb.so", "elfType" : 3, "buildId" : "6B82ECC04F262F87646E106FD82254D7B25684A6" }, { "b" : "7F17FA509000", "path" : "/lib64/libdb-4.7.so", "elfType" : 3, "buildId" : "E3E19783ECE59CE43F5C2EAB3F2529DFE798EB3B" }, { "b" : "7F17FA304000", "path" : "/usr/lib64/sasl2/libplain.so", "elfType" : 3, "buildId" : "916D3F3990557E30CB579ED3D144FA57774B9B6C" }, { "b" : "7F17FA0FF000", "path" : "/usr/lib64/sasl2/liblogin.so", "elfType" : 3, "buildId" : "7660C66EFF41AB59B1740F97C3E6F8D79D23567C" }, { "b" : "7F17F9EFA000", "path" : "/usr/lib64/sasl2/libanonymous.so", "elfType" : 3, "buildId" : "523C058C8D946374FF6651FE748617BAC2A37371" } ] }}
mongos(_ZN5mongo15printStackTraceERSo+0x26) [0xa98bb6]
mongos(+0x6982D2) [0xa982d2]
mongos(+0x69860E) [0xa9860e]
libpthread.so.0(+0xF5B0) [0x7f18029795b0]
mongos(_ZNK5mongo5Shard10runCommandERKSsRKNS_7BSONObjERS3_+0x5B) [0x97122b]
mongos(_ZNK5mongo5Shard10runCommandERKSsS2_RNS_7BSONObjE+0x15B) [0x971d1b]
mongos(+0x59B237) [0x99b237]
mongos(_ZN5mongo7Command22execCommandClientBasicEPNS_16OperationContextEPS0_RNS_11ClientBasicEiPKcRNS_7BSONObjERNS_14BSONObjBuilderE+0x4B8) [0x9deff8]
mongos(_ZN5mongo7Command20runAgainstRegisteredEPKcRNS_7BSONObjERNS_14BSONObjBuilderEi+0x26C) [0x9dfbbc]
mongos(_ZN5mongo8Strategy15clientCommandOpERNS_7RequestE+0x1B8) [0x9e57c8]
mongos(_ZN5mongo7Request7processEi+0x624) [0x9de624]
mongos(_ZN5mongo21ShardedMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortE+0x3E) [0x600ace]
mongos(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x261) [0xa518c1]
libpthread.so.0(+0x7F18) [0x7f1802971f18]
libc.so.6(clone+0x6D) [0x7f18026a7b2d]
----- END BACKTRACE -----