Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-22612

Invariant failure newCommittedSnapshot.opTime >= _currentCommittedSnapshot->opTime src/mongo/db/repl/replication_coordinator_impl.cpp

    XMLWordPrintable

    Details

    • Type: Bug
    • Status: Closed
    • Priority: Critical - P2
    • Resolution: Duplicate
    • Affects Version/s: 3.2.1
    • Fix Version/s: None
    • Component/s: Replication
    • Labels:
      None
    • Steps To Reproduce:
      Hide

      Replication set with 3 nodes running 24/7 with a lot of inserts and queries.

      Show
      Replication set with 3 nodes running 24/7 with a lot of inserts and queries.

      Description

      We have received the error below multiple times, resulting in the mongod process shutting down and the production system losing a node in the replication setup (we have 3 nodes).

      2016-02-11T21:43:10.516+0000 I -        [SnapshotThread] Invariant failure newCommittedSnapshot.opTime >= _currentCommittedSnapshot->opTime src/mongo/db/repl/replication_coordinator_impl.cpp 3228
      2016-02-11T21:43:10.516+0000 I -        [SnapshotThread]
       
      ***aborting after invariant() failure
       
       
      2016-02-11T21:43:10.551+0000 F -        [SnapshotThread] Got signal: 6 (Aborted).
       
       0x12d5772 0x12d46a9 0x12d4eb2 0x7fd855de4100 0x7fd855a485f7 0x7fd855a49ce8 0x125c83b 0xe96015 0xe9a596 0xe56a7b 0x1a99100 0x7fd855ddcdc5 0x7fd855b09bdd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"ED5772"},{"b":"400000","o":"ED46A9"},{"b":"400000","o":"ED4EB2"},{"b":"7FD855DD5000","o":"F100"},{"b":"7FD855A13000","o":"355F7"},{"b":"7FD855A13000","o":"36CE8"},{"b":"400000","o":"E5C83B"},{"b":"400000","o":"A96015"},{"b":"400000","o":"A9A596"},{"b":"400000","o":"A56A7B"},{"b":"400000","o":"1699100"},{"b":"7FD855DD5000","o":"7DC5"},{"b":"7FD855A13000","o":"F6BDD"}],"processInfo":{ "mongodbVersion" : "3.2.1", "gitVersion" : "a14d55980c2cdc565d4704a7e3ad37e4e535c1b2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.1.10-17.31.amzn1.x86_64", "version" : "#1 SMP Sat Oct 24 01:31:37 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "A931E563057BCC34ED2D1971AAC77D44F47033A9" }, { "b" : "7FFEDE7E0000", "elfType" : 3, "buildId" : "4106419F92DF72BDE396133D4FE9E47CB6983EF2" }, { "b" : "7FD856FFF000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "22480480235F3B1C6C2E5E5953949728676D3796" }, { "b" : "7FD856C1A000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F1C146B78505646930DD9003AA2B3622C5226D1B" }, { "b" : "7FD856A12000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "42833B65941483A611C40EA7D32F56EA83EA6E93" }, { "b" : "7FD85680E000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "6335077ACD51527BE9F2F18451A88E2B7350C5B6" }, { "b" : "7FD856509000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "0A90C35D3174805453EA67A785446D628E298B59" }, { "b" : "7FD856207000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "BB312C4A65B8FD830C148612CBEACEACC8B08E4F" }, { "b" : "7FD855FF1000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "00FA2883FB47B1327397BBF167C52F51A723D013" }, { "b" : "7FD855DD5000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "E5E575776DAD20ADE8FD0CAF17897C9D89020A87" }, { "b" : "7FD855A13000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "D84E3AFDFF3E164A09C125F85B5DCABC6F545B5E" }, { "b" : "7FD85726C000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "7B7BF8FEEF1A9C627EF90CA5C9188EFD4DA2DDD2" }, { "b" : "7FD8557C7000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "FF843C37C38E5BFFD57F7BCCAE05FDADC6390C8F" }, { "b" : "7FD8554E4000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "0BB150CC29DB5B0E039879EFC00152A75E3B00B9" }, { "b" : "7FD8552E1000", "path" : "/usr/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "5C01209C5AE1B1714F19B07EB58F2A1274B69DC8" }, { "b" : "7FD8550AF000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "1485992B0E5CDBA0A34817FC8C6A4C45E82CD1A9" }, { "b" : "7FD854E99000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "89C6AF118B6B4FB6A73AE1813E2C8BDD722956D1" }, { "b" : "7FD854C8A000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "A75A81EC50E9E0164A12B59D9987AD61AC7576C8" }, { "b" : "7FD854A87000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "37A58210FA50C91E09387765408A92909468D25B" }, { "b" : "7FD85486D000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "47EC2C63132D25E4FE83F77023DA1A66457A88F1" }, { "b" : "7FD85464C000", "path" : "/usr/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "F5054DC94443326819FBF3065CFDF5E4726F57EE" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x12d5772]
       mongod(+0xED46A9) [0x12d46a9]
       mongod(+0xED4EB2) [0x12d4eb2]
       libpthread.so.0(+0xF100) [0x7fd855de4100]
       libc.so.6(gsignal+0x37) [0x7fd855a485f7]
       libc.so.6(abort+0x148) [0x7fd855a49ce8]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x125c83b]
       mongod(+0xA96015) [0xe96015]
       mongod(_ZN5mongo4repl26ReplicationCoordinatorImpl16onSnapshotCreateENS0_6OpTimeENS_12SnapshotNameE+0xC6) [0xe9a596]
       mongod(_ZN5mongo4repl14SnapshotThread3runEv+0x4FB) [0xe56a7b]
       mongod(+0x1699100) [0x1a99100]
       libpthread.so.0(+0x7DC5) [0x7fd855ddcdc5]
       libc.so.6(clone+0x6D) [0x7fd855b09bdd]
      -----  END BACKTRACE  -----
      

      Line where assertion fails:

      https://github.com/mongodb/mongo/blob/a14d55980c2cdc565d4704a7e3ad37e4e535c1b2/src/mongo/db/repl/replication_coordinator_impl.cpp#L3228

      MongoDB configuration:

      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] MongoDB starting : pid=10039 port=27017 dbpath=/var/lib/mongo 64-bit host=mongodb-2.sentinelengine.com
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] db version v3.2.1
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] git version: a14d55980c2cdc565d4704a7e3ad37e4e535c1b2
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.0.0-fips 29 Mar 2010
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] allocator: tcmalloc
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] modules: none
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] build environment:
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten]     distmod: amazon
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten]     distarch: x86_64
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten]     target_arch: x86_64
      2016-02-15T09:07:09.395+0000 I CONTROL  [initandlisten] options: { config: "/etc/mongod.conf", net: { port: 27017 }, processManagement: { fork: true, pidFilePath: "/var/run/mongodb/mongod.pid" }, replication: { enableMajorityReadConcern: true, replSetName: "rs0" }, security: { authorization: "enabled", keyFile: "/srv/mongodb/mongodb-keyfile" }, storage: { dbPath: "/var/lib/mongo", engine: "wiredTiger", journal: { enabled: true } }, systemLog: { destination: "file", logAppend: true, path: "/var/log/mongodb/mongod.log" } }
      

      Machine specification:

          "ansible_facts": {
              "ansible_architecture": "x86_64",
              "ansible_bios_date": "12/07/2015",
              "ansible_bios_version": "4.2.amazon",
              "ansible_default_ipv6": {},
              "ansible_distribution": "Amazon",
              "ansible_distribution_version": "2015.09",
              "ansible_kernel": "4.1.10-17.31.amzn1.x86_64",
              "ansible_machine": "x86_64",
              "ansible_memtotal_mb": 3955,
              "ansible_os_family": "RedHat",
              "ansible_processor": [
                  "GenuineIntel",
                  "Intel(R) Xeon(R) CPU E5-2676 v3 @ 2.40GHz",
                  "GenuineIntel",
                  "Intel(R) Xeon(R) CPU E5-2676 v3 @ 2.40GHz"
              ],
              "ansible_processor_cores": 2,
              "ansible_processor_count": 2,
              "ansible_processor_threads_per_core": 1,
              "ansible_processor_vcpus": 2,
              "ansible_selinux": false
          }
      

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              Unassigned
              Reporter:
              jonas.lalin@besedo.com Jonas Lalin
              Participants:
              Votes:
              0 Vote for this issue
              Watchers:
              4 Start watching this issue

                Dates

                Created:
                Updated:
                Resolved: