• Type: Icon: Bug Bug
    • Resolution: Incomplete
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.0.8
    • Component/s: Replication, WiredTiger
    • Labels:
      None
    • ALL

      In a 3 node replica set, on of the secondary members just crashed after months of working correctly with the following stack trace:

      2016-04-08T02:58:33.316-0500 F -        Invalid access at address: 0
      2016-04-08T02:58:33.403-0500 F -        Got signal: 11 (Segmentation fault).
      
       0xf9a212 0xf99873 0xf99bd4 0x7f99137d9710 0xffd043 0xffd11d 0x146a200 0x1357402 0x1368157 0x13a6fb0 0x13a47db 0x13a4c89 0x13a4e6a 0x7f99137d19d1 0x7f99123238fd
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B9A212"},{"b":"400000","o":"B99873"},{"b":"400000","o":"B99BD4"},{"b":"7F99137CA000","o":"F710"},{"b":"400000","o":"BFD043"},{"b":"400000","o":"BFD11D"},{"b":"400000","o":"106A200"},{"b":"400000","o":"F57402"},{"b":"400000","o":"F68157"},{"b":"400000","o":"FA6FB0"},{"b":"400000","o":"FA47DB"},{"b":"400000","o":"FA4C89"},{"b":"400000","o":"FA4E6A"},{"b":"7F99137CA000","o":"79D1"},{"b":"7F991223B000","o":"E88FD"}],"processInfo":{ "mongodbVersion" : "3.0.8", "gitVersion" : "83d8cc25e00e42856924d84e220fbe4a839e605d", "uname" : { "sysname" : "Linux", "release" : "2.6.32-504.38.1.el6.x86_64", "version" : "#1 SMP Sun Oct 4 13:43:08 EDT 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "1CD29DAD308AB5BA5360712532A856503A3012F9" }, { "b" : "7FFDEC7DD000", "elfType" : 3, "buildId" : "0DCE83815830C012D31AC0E3B51A0FCB409C7C97" }, { "b" : "7F99137CA000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "A35053D76A6B7BD91D2EE58CC024D8EF697CE977" }, { "b" : "7F991355E000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "6BE75D1A76F11E2D7C82CB505A28C4D1A6E31F4D" }, { "b" : "7F991317B000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "129F9ADB68579E5BEE43511F44A2DAC805A5BF67" }, { "b" : "7F9912F73000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "69BCB2B5FE6D85ACD898362EAC5EE79857DA4EC4" }, { "b" : "7F9912D6F000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "266172B083F783BD94389BE55B0B371C17198268" }, { "b" : "7F9912A69000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "743EA30ADF8E973D45AB59200C307F5ABC2749F6" }, { "b" : "7F99127E5000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "F4B82F632E515FFA38AA9202FAC200ACD78BCCE6" }, { "b" : "7F99125CF000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "7918143A0396110395C28377A1F202C769EFAC65" }, { "b" : "7F991223B000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "C7DF056B7C109A41096296CD70702F2EADA124B0" }, { "b" : "7F99139E7000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5BEB2450B75E84FF317C65F22AF8B8112C25DF63" }, { "b" : "7F9911FF7000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "7158754011EC3ECEE4D7C09675EA8AEB3EC4B909" }, { "b" : "7F9911D11000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "9C84EC93B5FC6E3D72F81E75B368E63ADF38715F" }, { "b" : "7F9911B0D000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "6A22EDFF4D4F04A57573E3D1536B6B4963159CD5" }, { "b" : "7F99118E1000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A00CE71A59B7B771E839114DF00DF28DC660B645" }, { "b" : "7F99116CB000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "D053BB4FF0C2FC983842F81598813B9B931AD0D1" }, { "b" : "7F99114C0000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "021F66AE06126AC8910B7A2189EC13D1E036849C" }, { "b" : "7F99112BD000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "3BCCABE75DC61BBA81AAE45D164E26EF4F9F55DB" }, { "b" : "7F99110A3000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C489246DFBF195A5557C1067E76504B8EDB23D41" }, { "b" : "7F9910E84000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "2D0F26E648D9661ABD83ED8B4BBE8F2CFA50393B" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xf9a212]
       mongod(+0xB99873) [0xf99873]
       mongod(+0xB99BD4) [0xf99bd4]
       libpthread.so.0(+0xF710) [0x7f99137d9710]
       mongod(_ZN8tcmalloc11ThreadCache21ReleaseToCentralCacheEPNS0_8FreeListEmi+0xE3) [0xffd043]
       mongod(_ZN8tcmalloc11ThreadCache11ListTooLongEPNS0_8FreeListEm+0x1D) [0xffd11d]
       mongod(free+0x1F0) [0x146a200]
       mongod(__wt_page_out+0x282) [0x1357402]
       mongod(__wt_split_multi+0x2E7) [0x1368157]
       mongod(__wt_evict+0x7C0) [0x13a6fb0]
       mongod(__wt_evict_page+0x3B) [0x13a47db]
       mongod(__wt_evict_lru_page+0x249) [0x13a4c89]
       mongod(+0xFA4E6A) [0x13a4e6a]
       libpthread.so.0(+0x79D1) [0x7f99137d19d1]
       libc.so.6(clone+0x6D) [0x7f99123238fd]
      -----  END BACKTRACE  -----
      

      Running mongo 3.0.8 with wiredTiger on all 3 nodes.

      I've seen similar issues a few times on your forum, but they were all closed in version 3.0.8.

      Any idea what might have caused this?

            Assignee:
            Unassigned Unassigned
            Reporter:
            rburczyk@solution-tek.com Robert Burczyk
            Votes:
            0 Vote for this issue
            Watchers:
            10 Start watching this issue

              Created:
              Updated:
              Resolved: