Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-15797

[TTLMonitor] Invariant failure - server crashes on start.

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Critical - P2 Critical - P2
    • 2.8.0-rc0
    • Affects Version/s: 2.7.8
    • Component/s: Stability
    • Labels:
      None
    • ALL
    • Hide

      Restart server in replica set (only one member)

      Show
      Restart server in replica set (only one member)

      Here is a log dump pre/post crash. Happens every time.

      2014-10-24T13:31:23.979+0200 I JOURNAL  [initandlisten] preallocating a journal file /var/lib/mongodb/journal/prealloc.0
      2014-10-24T13:31:26.654+0200 I -        [initandlisten]   File Preallocator Progress: 996147200/1073741824 92%
      2014-10-24T13:31:35.649+0200 I NETWORK  [websvr] admin web console waiting for connections on port 28017
      2014-10-24T13:31:35.695+0200 I REPLSETS [rsSync] replSet warning did not receive a valid config yet, sleeping 5 seconds 
      2014-10-24T13:31:35.713+0200 I REPLSETS transition to STARTUP2
      2014-10-24T13:31:35.714+0200 I NETWORK  [initandlisten] waiting for connections on port 27017
      2014-10-24T13:31:36.696+0200 I REPLSETS replSet could not find member to sync from
      2014-10-24T13:31:40.696+0200 I REPLSETS transition to RECOVERING
      2014-10-24T13:31:40.721+0200 I REPLSETS transition to PRIMARY
      2014-10-24T13:31:40.724+0200 I REPLSETS [rsSync] transition to primary complete; database writes are now permitted
      2014-10-24T13:31:53.666+0200 I NETWORK  [initandlisten] connection accepted from 127.0.0.1:59245 #1 (1 connection now open)
      2014-10-24T13:32:23.110+0200 I NETWORK  [conn1] end connection 127.0.0.1:59245 (0 connections now open)
      2014-10-24T13:32:35.737+0200 I -        [TTLMonitor] Invariant failure !inAWriteUnitOfWork() src/mongo/db/concurrency/lock_state.cpp 572
      2014-10-24T13:32:35.746+0200 I -        [TTLMonitor] 
       0xf260f9 0xed1ed1 0xeb89b3 0x9b29a2 0x9b6790 0xbc104f 0xbbd3a6 0xbbe11d 0xae078a 0xadfa40 0xd5fbb3 0xd6028d 0xebb7fc 0xf72844 0x7fba85813182 0x7fba84913fbd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B260F9"},{"b":"400000","o":"AD1ED1"},{"b":"400000","o":"AB89B3"},{"b":"400000","o":"5B29A2"},{"b":"400000","o":"5B6790"},{"b":"400000","o":"7C104F"},{"b":"400000","o":"7BD3A6"},{"b":"400000","o":"7BE11D"},{"b":"400000","o":"6E078A"},{"b":"400000","o":"6DFA40"},{"b":"400000","o":"95FBB3"},{"b":"400000","o":"96028D"},{"b":"400000","o":"ABB7FC"},{"b":"400000","o":"B72844"},{"b":"7FBA8580B000","o":"8182"},{"b":"7FBA84819000","o":"FAFBD"}],"processInfo":{ "mongodbVersion" : "2.7.8", "gitVersion" : "0e9cb3b20498b9f167afaff7a5c4a4d1da7e06a2", "uname" : { "sysname" : "Linux", "release" : "3.13.0-35-generic", "version" : "#62-Ubuntu SMP Fri Aug 15 01:58:42 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF018FE000", "elfType" : 3 }, { "b" : "7FBA8580B000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7FBA85603000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7FBA853FF000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7FBA850FB000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7FBA84DF5000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7FBA84BDF000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7FBA84819000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7FBA85A29000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf260f9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xed1ed1]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0xD3) [0xeb89b3]
       mongod(_ZN5mongo10LockerImplILb1EE22saveLockStateAndUnlockEPNS_6Locker12LockSnapshotE+0x3D2) [0x9b29a2]
       mongod(_ZN5mongo5Yield13yieldAllLocksEPNS_16OperationContextEi+0x60) [0x9b6790]
       mongod(_ZN5mongo15PlanYieldPolicy5yieldEb+0x5F) [0xbc104f]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_7DiskLocE+0x66) [0xbbd3a6]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbbe11d]
       mongod(_ZN5mongo14DeleteExecutor7executeEPNS_8DatabaseE+0x5A) [0xae078a]
       mongod(_ZN5mongo13deleteObjectsEPNS_16OperationContextEPNS_8DatabaseERKNS_10StringDataENS_7BSONObjENS_12PlanExecutor11YieldPolicyEbbbb+0x190) [0xadfa40]
       mongod(_ZN5mongo10TTLMonitor13doTTLForIndexEPNS_16OperationContextERKSsRKNS_7BSONObjE+0x9A3) [0xd5fbb3]
       mongod(_ZN5mongo10TTLMonitor3runEv+0x38D) [0xd6028d]
       mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x11C) [0xebb7fc]
       mongod(+0xB72844) [0xf72844]
       libpthread.so.0(+0x8182) [0x7fba85813182]
       libc.so.6(clone+0x6D) [0x7fba84913fbd]
      -----  END BACKTRACE  -----
      2014-10-24T13:32:35.746+0200 I -        [TTLMonitor] 
      
      ***aborting after invariant() failure
      
      
      2014-10-24T13:32:35.755+0200 F -        [TTLMonitor] Got signal: 6 (Aborted).
      
       0xf260f9 0xf25cb2 0xf26036 0x7fba8484fc30 0x7fba8484fbb9 0x7fba84852fc8 0xeb89f8 0x9b29a2 0x9b6790 0xbc104f 0xbbd3a6 0xbbe11d 0xae078a 0xadfa40 0xd5fbb3 0xd6028d 0xebb7fc 0xf72844 0x7fba85813182 0x7fba84913fbd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B260F9"},{"b":"400000","o":"B25CB2"},{"b":"400000","o":"B26036"},{"b":"7FBA84819000","o":"36C30"},{"b":"7FBA84819000","o":"36BB9"},{"b":"7FBA84819000","o":"39FC8"},{"b":"400000","o":"AB89F8"},{"b":"400000","o":"5B29A2"},{"b":"400000","o":"5B6790"},{"b":"400000","o":"7C104F"},{"b":"400000","o":"7BD3A6"},{"b":"400000","o":"7BE11D"},{"b":"400000","o":"6E078A"},{"b":"400000","o":"6DFA40"},{"b":"400000","o":"95FBB3"},{"b":"400000","o":"96028D"},{"b":"400000","o":"ABB7FC"},{"b":"400000","o":"B72844"},{"b":"7FBA8580B000","o":"8182"},{"b":"7FBA84819000","o":"FAFBD"}],"processInfo":{ "mongodbVersion" : "2.7.8", "gitVersion" : "0e9cb3b20498b9f167afaff7a5c4a4d1da7e06a2", "uname" : { "sysname" : "Linux", "release" : "3.13.0-35-generic", "version" : "#62-Ubuntu SMP Fri Aug 15 01:58:42 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000" }, { "b" : "7FFF018FE000", "elfType" : 3 }, { "b" : "7FBA8580B000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3 }, { "b" : "7FBA85603000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3 }, { "b" : "7FBA853FF000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3 }, { "b" : "7FBA850FB000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3 }, { "b" : "7FBA84DF5000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3 }, { "b" : "7FBA84BDF000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3 }, { "b" : "7FBA84819000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3 }, { "b" : "7FBA85A29000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3 } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf260f9]
       mongod(+0xB25CB2) [0xf25cb2]
       mongod(+0xB26036) [0xf26036]
       libc.so.6(+0x36C30) [0x7fba8484fc30]
       libc.so.6(gsignal+0x39) [0x7fba8484fbb9]
       libc.so.6(abort+0x148) [0x7fba84852fc8]
       mongod(_ZN5mongo15invariantFailedEPKcS1_j+0x118) [0xeb89f8]
       mongod(_ZN5mongo10LockerImplILb1EE22saveLockStateAndUnlockEPNS_6Locker12LockSnapshotE+0x3D2) [0x9b29a2]
       mongod(_ZN5mongo5Yield13yieldAllLocksEPNS_16OperationContextEi+0x60) [0x9b6790]
       mongod(_ZN5mongo15PlanYieldPolicy5yieldEb+0x5F) [0xbc104f]
       mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_7DiskLocE+0x66) [0xbbd3a6]
       mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbbe11d]
       mongod(_ZN5mongo14DeleteExecutor7executeEPNS_8DatabaseE+0x5A) [0xae078a]
       mongod(_ZN5mongo13deleteObjectsEPNS_16OperationContextEPNS_8DatabaseERKNS_10StringDataENS_7BSONObjENS_12PlanExecutor11YieldPolicyEbbbb+0x190) [0xadfa40]
       mongod(_ZN5mongo10TTLMonitor13doTTLForIndexEPNS_16OperationContextERKSsRKNS_7BSONObjE+0x9A3) [0xd5fbb3]
       mongod(_ZN5mongo10TTLMonitor3runEv+0x38D) [0xd6028d]
       mongod(_ZN5mongo13BackgroundJob7jobBodyEv+0x11C) [0xebb7fc]
       mongod(+0xB72844) [0xf72844]
       libpthread.so.0(+0x8182) [0x7fba85813182]
       libc.so.6(clone+0x6D) [0x7fba84913fbd]
      -----  END BACKTRACE  -----
      

      Please let me know what else you need from me to pinpoint this.

            Assignee:
            rassi J Rassi
            Reporter:
            pieterwjordaanpc@gmail.com Pieter Jordaan
            Votes:
            0 Vote for this issue
            Watchers:
            4 Start watching this issue

              Created:
              Updated:
              Resolved: