Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-40355

rs.config that contains an _id greater than the number of nodes will crash

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical - P2
    • Resolution: Fixed
    • Affects Version/s: 4.0.7
    • Fix Version/s: 3.6.15, 4.0.8, 4.1.10, 3.4.24
    • Component/s: Replication
    • Labels:
      None
    • Backwards Compatibility:
      Fully Compatible
    • Operating System:
      ALL
    • Backport Requested:
      v4.0, v3.6, v3.4
    • Steps To Reproduce:
      Hide

      (function() {
          "use strict";
       
          const name = "arbiters_not_included_in_w2_wc";
          const rst = new ReplSetTest({name: name, nodes: 3});
          const nodes = rst.nodeList();
       
          rst.startSet();
          rst.initiate({
              "_id": name,
              "members": [
                  {"_id": 1, "host": nodes[0]},
                  {"_id": 3, "host": nodes[1]},
                  {"_id": 5, "host": nodes[2]},
              ]
          });
       
          const dbName = "test";
          const collName = name;
       
          const primary = rst.getPrimary();
          const testColl = primary.getDB(dbName).getCollection(collName);
       
          assert.commandWorked(
              testColl.insert({"a": 1}, {writeConcern: {w: 2}}));
      })();
      

      Show
      ( function () { "use strict" ;   const name = "arbiters_not_included_in_w2_wc" ; const rst = new ReplSetTest({name: name, nodes: 3}); const nodes = rst.nodeList();   rst.startSet(); rst.initiate({ "_id" : name, "members" : [ { "_id" : 1, "host" : nodes[0]}, { "_id" : 3, "host" : nodes[1]}, { "_id" : 5, "host" : nodes[2]}, ] });   const dbName = "test" ; const collName = name;   const primary = rst.getPrimary(); const testColl = primary.getDB(dbName).getCollection(collName);   assert.commandWorked( testColl.insert({ "a" : 1}, {writeConcern: {w: 2}})); })();
    • Sprint:
      Repl 2019-04-08
    • Case:

      Description

      SERVER-37846 introduced this check that incorrectly tries to look up the MemberConfig by passing the configId to getMemberAt, which actually operates on an array index id.

      When this occurs the following crash is logged:

      2019-03-27T00:50:37.868+0000 D REPL     [conn49] Waiting for write concern. OpTime: { ts: Timestamp(1553647837, 1), t: 7921 }, write concern: { w: 2, wtimeout: 0 }
      2019-03-27T00:50:37.868+0000 F -        [conn49] Invariant failure i < _members.size() src/mongo/db/repl/repl_set_config.cpp 660
      2019-03-27T00:50:37.868+0000 F -        [conn49] 
       
      ***aborting after invariant() failure
       
       
      2019-03-27T00:50:37.890+0000 F -        [conn49] Got signal: 6 (Aborted).
       0x55b294737ce1 0x55b294736ef9 0x55b2947373dd 0x7fb14c6fd390 0x7fb14c357428 0x7fb14c35902a 0x55b292d460ac 0x55b293fed08c 0x55b293fed1e9 0x55b29309144c 0x55b29305e6e7 0x55b29306531e 0x55b293065904 0x55b29342b78b 0x55b292de4e43 0x55b292df43ed 0x55b292df67db 0x55b292df80f9 0x55b292df9031 0x55b292de478a 0x55b292df129a 0x55b292dec017 0x55b292def771 0x55b293f90122 0x55b292dea200 0x55b292ded2f5 0x55b292deb757 0x55b292dec09d 0x55b292def771 0x55b293f90685 0x55b294690794 0x7fb14c6f36ba 0x7fb14c42941d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"55B29232A000","o":"240DCE1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55B29232A000","o":"240CEF9"},{"b":"55B29232A000","o":"240D3DD"},{"b":"7FB14C6EC000","o":"11390"},{"b":"7FB14C322000","o":"35428","s":"gsignal"},{"b":"7FB14C322000","o":"3702A","s":"abort"},{"b":"55B29232A000","o":"A1C0AC","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j"},{"b":"55B29232A000","o":"1CC308C"},{"b":"55B29232A000","o":"1CC31E9"},{"b":"55B29232A000","o":"D6744C","s":"_ZN5mongo4repl19TopologyCoordinator25haveNumNodesReachedOpTimeERKNS0_6OpTimeEib"},{"b":"55B29232A000","o":"D346E7","s":"_ZN5mongo4repl26ReplicationCoordinatorImpl33_doneWaitingForReplication_inlockERKNS0_6OpTimeERKNS_19WriteConcernOptionsE"},{"b":"55B29232A000","o":"D3B31E","s":"_ZN5mongo4repl26ReplicationCoordinatorImpl24_awaitReplication_inlockEPSt11unique_lockISt5mutexEPNS_16OperationContextERKNS0_6OpTimeERKNS_19WriteConcernOptionsE"},{"b":"55B29232A000","o":"D3B904","s":"_ZN5mongo4repl26ReplicationCoordinatorImpl16awaitReplicationEPNS_16OperationContextERKNS0_6OpTimeERKNS_19WriteConcernOptionsE"},{"b":"55B29232A000","o":"110178B","s":"_ZN5mongo19waitForWriteConcernEPNS_16OperationContextERKNS_4repl6OpTimeERKNS_19WriteConcernOptionsEPNS_18WriteConcernResultE"},{"b":"55B29232A000","o":"ABAE43","s":"_ZNK5mongo23ServiceEntryPointMongod5Hooks19waitForWriteConcernEPNS_16OperationContextEPKNS_17CommandInvocationERKNS_4repl6OpTimeERNS_14BSONObjBuilderE"},{"b":"55B29232A000","o":"ACA3ED"},{"b":"55B29232A000","o":"ACC7DB"},{"b":"55B29232A000","o":"ACE0F9"},{"b":"55B29232A000","o":"ACF031","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},{"b":"55B29232A000","o":"ABA78A","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},{"b":"55B29232A000","o":"AC729A","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},{"b":"55B29232A000","o":"AC2017","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55B29232A000","o":"AC5771"},{"b":"55B29232A000","o":"1C66122","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},{"b":"55B29232A000","o":"AC0200","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},{"b":"55B29232A000","o":"AC32F5","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},{"b":"55B29232A000","o":"AC1757","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},{"b":"55B29232A000","o":"AC209D","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},{"b":"55B29232A000","o":"AC5771"},{"b":"55B29232A000","o":"1C66685"},{"b":"55B29232A000","o":"2366794"},{"b":"7FB14C6EC000","o":"76BA"},{"b":"7FB14C322000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.7", "gitVersion" : "1b82c812a9c0bbf6dc79d5400de9ea99e6ffa025", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.0-143-generic", "version" : "#169-Ubuntu SMP Thu Feb 7 07:56:38 UTC 2019", "machine" : "x86_64" }, "somap" : [ { "b" : "55B29232A000", "elfType" : 3, "buildId" : "E19C588F8809059D31942581C2F07E30CEFAD52A" }, { "b" : "7FFC490E0000", "elfType" : 3, "buildId" : "71EA263515008587FA2EC6DD7EA6B96ACA79DDDB" }, { "b" : "7FB14DAFD000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "93EA3FEB7AC8F28D7F4F59F5B94226A20A6F9023" }, { "b" : "7FB14D8E2000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "50A923F8DAFECBCD969C8573116A38C18D0E24D5" }, { "b" : "7FB14D49D000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "15FFEB43278726B025F020862BF51302822A40EC" }, { "b" : "7FB14D234000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "FF69EA60EBE05F2DD689D2B26FC85A73E5FBC3A0" }, { "b" : "7FB14D030000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "37BFC3D8F7E3B022DAC7943B1A5FACD40CEBF0AD" }, { "b" : "7FB14CE28000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "69143E8B39040C964D3958490535322675F15DD3" }, { "b" : "7FB14CB1F000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "BAD67A84E56E73D031AE507261DA066B35949D34" }, { "b" : "7FB14C909000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FB14C6EC000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "B17C21299099640A6D863E423D99265824E7BB16" }, { "b" : "7FB14C322000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "1CA54A6E0D76188105B12E49FE6B8019BF08803A" }, { "b" : "7FB14DD6C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "C0ADBAD6F9A33944F2B3567C078EC472A1DAE98E" }, { "b" : "7FB14C0EF000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FB14BED3000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FB14BC89000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "41971A4A3CCDC54A447F41DF4BD96C948C546E0E" }, { "b" : "7FB14BA7A000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FB14B829000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FB14B60F000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FB14B2DF000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FB14B0AC000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FB14AE76000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FB14ABF6000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FB14A924000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "0EEF7058B0737B68BDF89E5DC604D0AC389C8BB1" }, { "b" : "7FB14A6F5000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "FFBA483A43D9EF73925AC116811890C037523DA1" }, { "b" : "7FB14A4F1000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FB14A2E6000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "B789D8D4B4FC333405AB34387D9237F954060EA4" }, { "b" : "7FB14A0CB000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FB149E8A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FB149C26000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FB149A13000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FB14980F000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FB149606000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FB14937C000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FB1490DA000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FB148EA7000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FB148C91000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FB148A89000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FB148860000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FB148651000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FB148406000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FB148132000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "C646E090F27A5CAEE45AC2BE4EFBC01E550BD27B" }, { "b" : "7FB147EFA000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "FD61CA7A6D603E94E5EFD5C88D8810AE104FCF40" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55b294737ce1]
       mongod(+0x240CEF9) [0x55b294736ef9]
       mongod(+0x240D3DD) [0x55b2947373dd]
       libpthread.so.0(+0x11390) [0x7fb14c6fd390]
       libc.so.6(gsignal+0x38) [0x7fb14c357428]
       libc.so.6(abort+0x16A) [0x7fb14c35902a]
       mongod(_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j+0x0) [0x55b292d460ac]
       mongod(+0x1CC308C) [0x55b293fed08c]
       mongod(+0x1CC31E9) [0x55b293fed1e9]
       mongod(_ZN5mongo4repl19TopologyCoordinator25haveNumNodesReachedOpTimeERKNS0_6OpTimeEib+0xDC) [0x55b29309144c]
       mongod(_ZN5mongo4repl26ReplicationCoordinatorImpl33_doneWaitingForReplication_inlockERKNS0_6OpTimeERKNS_19WriteConcernOptionsE+0x137) [0x55b29305e6e7]
       mongod(_ZN5mongo4repl26ReplicationCoordinatorImpl24_awaitReplication_inlockEPSt11unique_lockISt5mutexEPNS_16OperationContextERKNS0_6OpTimeERKNS_19WriteConcernOptionsE+0x33E) [0x55b29306531e]
       mongod(_ZN5mongo4repl26ReplicationCoordinatorImpl16awaitReplicationEPNS_16OperationContextERKNS0_6OpTimeERKNS_19WriteConcernOptionsE+0x134) [0x55b293065904]
       mongod(_ZN5mongo19waitForWriteConcernEPNS_16OperationContextERKNS_4repl6OpTimeERKNS_19WriteConcernOptionsEPNS_18WriteConcernResultE+0x39B) [0x55b29342b78b]
       mongod(_ZNK5mongo23ServiceEntryPointMongod5Hooks19waitForWriteConcernEPNS_16OperationContextEPKNS_17CommandInvocationERKNS_4repl6OpTimeERNS_14BSONObjBuilderE+0x103) [0x55b292de4e43]
       mongod(+0xACA3ED) [0x55b292df43ed]
       mongod(+0xACC7DB) [0x55b292df67db]
       mongod(+0xACE0F9) [0x55b292df80f9]
       mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x3C1) [0x55b292df9031]
       mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3A) [0x55b292de478a]
       mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x55b292df129a]
       mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x55b292dec017]
       mongod(+0xAC5771) [0x55b292def771]
       mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x1A2) [0x55b293f90122]
       mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x150) [0x55b292dea200]
       mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xB05) [0x55b292ded2f5]
       mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x55b292deb757]
       mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x55b292dec09d]
       mongod(+0xAC5771) [0x55b292def771]
       mongod(+0x1C66685) [0x55b293f90685]
       mongod(+0x2366794) [0x55b294690794]
       libpthread.so.0(+0x76BA) [0x7fb14c6f36ba]
       libc.so.6(clone+0x6D) [0x7fb14c42941d]
      -----  END BACKTRACE  -----
      

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              jesse A. Jesse Jiryu Davis
              Reporter:
              john.murphy John Murphy
              Participants:
              Votes:
              0 Vote for this issue
              Watchers:
              26 Start watching this issue

                Dates

                Created:
                Updated:
                Resolved: