Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-22617

SnapshotThread hits invariant due to reading oplog entries out of order

    • Fully Compatible
    • ALL
    • Integration 10 (02/22/16)

      it happens occasionaly in our production environment

      2016-02-08T16:21:02.045-0500 I -        [SnapshotThread] Invariant failure newCommittedSnapshot.opTime >= _currentCommittedSnapshot->opTime src/mongo/db/repl/replication_coordinator_impl.cpp 3228
      2016-02-08T16:21:02.045-0500 I -        [SnapshotThread] 
      
      ***aborting after invariant() failure
      
      
      2016-02-08T16:21:02.050-0500 F -        [SnapshotThread] Got signal: 6 (Aborted).
      
       0x12eaa52 0x12e9bb9 0x12ea3c2 0x7fdb551a3100 0x7fdb54e085f7 0x7fdb54e09ce8 0x127541b 0xecb495 0xecfa16 0xe90a4b 0x7fdb5597d220 0x7fdb5519bdc5 0x7fdb54ec91cd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"EEAA52"},{"b":"400000","o":"EE9BB9"},{"b":"400000","o":"EEA3C2"},{"b":"7FDB55194000","o":"F100"},{"b":"7FDB54DD3000","o":"355F7"},{"b":"7FDB54DD3000","o":"36CE8"},{"b":"400000","o":"E7541B"},{"b":"400000","o":"ACB495"},{"b":"400000","o":"ACFA16"},{"b":"400000","o":"A90A4B"},{"b":"7FDB558C8000","o":"B5220"},{"b":"7FDB55194000","o":"7DC5"},{"b":"7FDB54DD3000","o":"F61CD"}],"processInfo":{ "mongodbVersion" : "3.2.1", "gitVersion" : "a14d55980c2cdc565d4704a7e3ad37e4e535c1b2", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-327.el7.x86_64", "version" : "#1 SMP Thu Oct 29 17:29:29 EDT 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "82DE04175D9E4EDD5FB2C7106D67B02C808D491C" }, { "b" : "7FFD3EACB000", "elfType" : 3, "buildId" : "17A121B1F7BBB010F54735FFDE3347B27B33884D" }, { "b" : "7FDB563C4000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "8DB4545998776514159031B754BB67F7F396F83A" }, { "b" : "7FDB55FDD000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "038F79F7C3F6E60C29184B8E70D0B1E62525D64D" }, { "b" : "7FDB55DD5000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "A1D9E0B471D827008C36FA72BAB34BE08FE54B33" }, { "b" : "7FDB55BD1000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "5958E57738366BCC217783F3CD4C836437F7C45F" }, { "b" : "7FDB558C8000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "74AAD82C77F6D211BD9E4DA27BF4F62A0D938C4B" }, { "b" : "7FDB555C6000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "02C4E38A8145201D9C574499CF75132551835CEB" }, { "b" : "7FDB553B0000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "97D5E2F5739B715C3A0EC9F95F7336E232346CA8" }, { "b" : "7FDB55194000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "FA15B7D2CA650B34E6A0C9AD999BA6625AEC4068" }, { "b" : "7FDB54DD3000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "B0A1DFA62C6AF7AA62487E3C260DC4B9C24D8BF8" }, { "b" : "7FDB56631000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CEB78DAE1EE5B4C544047DC26F88A8E4586A34D2" }, { "b" : "7FDB54B87000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "8AB5682155DE13D0916B984306B4E044E216B2EB" }, { "b" : "7FDB548A2000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "BE8968836D439581B2816CE3827642FCF4B8BF4A" }, { "b" : "7FDB5469E000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "B25574847B066A26CD593C8101DF6779898FF2C2" }, { "b" : "7FDB5446C000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "F5784ED7E64118BAFE898DBF178DC9E37CBDA4AA" }, { "b" : "7FDB54256000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "FC37913FB197B822BCDBF3697D061E248698CEC1" }, { "b" : "7FDB54047000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BBED12CFDC9647C8771A4B897E0D5A4F217ED7C" }, { "b" : "7FDB53E43000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "8CA73C16CFEB9A8B5660015B9223B09F87041CAD" }, { "b" : "7FDB53C29000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "D08CF135D143704DA93E5F025AE6AE6943838F03" }, { "b" : "7FDB53A04000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "5062031216B995004A297D555D834C0109F7598C" }, { "b" : "7FDB537A3000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "8E3819A80BF876382A6F0CB2A08F82F1742EE8DB" }, { "b" : "7FDB5357E000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "61D7D46225E85F144221E1424B87FBF3CB2B9D3F" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x12eaa52]
       mongod(+0xEE9BB9) [0x12e9bb9]
       mongod(+0xEEA3C2) [0x12ea3c2]
       libpthread.so.0(+0xF100) [0x7fdb551a3100]
       libc.so.6(gsignal+0x37) [0x7fdb54e085f7]
       libc.so.6(abort+0x148) [0x7fdb54e09ce8]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xCB) [0x127541b]
       mongod(+0xACB495) [0xecb495]
       mongod(_ZN5mongo4repl26ReplicationCoordinatorImpl16onSnapshotCreateENS0_6OpTimeENS_12SnapshotNameE+0xC6) [0xecfa16]
       mongod(_ZN5mongo4repl14SnapshotThread3runEv+0x4FB) [0xe90a4b]
       libstdc++.so.6(+0xB5220) [0x7fdb5597d220]
       libpthread.so.0(+0x7DC5) [0x7fdb5519bdc5]
       libc.so.6(clone+0x6D) [0x7fdb54ec91cd]
      -----  END BACKTRACE  -----
      

      Steps to repro from orig ticket

      I'm not sure how to reproduce it.
      I have a replica set with 4 nodes running mongodb version 3.2.1 at AWS
      i2.8xlarge - linux - database on ephemeral storage
      i2.4xlarge - windows - database on ephemeral storage
      r2.2xlarge - windows - database on EBS
      t2.small - windows - arbiter
      I will be happy to explain any other details you might need
      

            Assignee:
            ramon.fernandez@mongodb.com Ramon Fernandez Marina
            Reporter:
            ricardo_fanatee Ricardo Hilsenrath
            Votes:
            0 Vote for this issue
            Watchers:
            24 Start watching this issue

              Created:
              Updated:
              Resolved: