Major - P3 Was trying to run a replica set test mirroring from a WT primary to each of a WT secondary and mmap secondary, running hammer. After about 7 hours, the mmap secondary crashed.
AWS, Ubuntu 14.04, 3.13.0-48-generic
db version v3.1.4-pre-
git version: cfe82d032b02a68cfd4a60ef4163d985a08c8018
2015-05-28T10:56:31.050+0000 I NETWORK [initandlisten] connection accepted from 74.113.166.198:51966 #30514 (37 connections now open)
2015-05-28T10:56:32.490+0000 I NETWORK [initandlisten] connection accepted from 74.113.166.198:51973 #30515 (38 connections now open)
2015-05-28T10:56:32.644+0000 I NETWORK [initandlisten] connection accepted from 74.113.166.198:51974 #30516 (39 connections now open)
2015-05-28T10:56:34.637+0000 I NETWORK [conn30516] end connection 74.113.166.198:51974 (38 connections now open)
2015-05-28T10:56:34.639+0000 I NETWORK [conn30515] end connection 74.113.166.198:51973 (37 connections now open)
2015-05-28T10:56:34.645+0000 I NETWORK [conn30514] end connection 74.113.166.198:51966 (36 connections now open)
2015-05-28T10:56:35.383+0000 F - [journal writer] Invalid access at address: 0x7ee1b8f1c0b0
2015-05-28T10:56:35.435+0000 F - [journal writer] Got signal: 7 (Bus error).
0x100dc86 0x100d3a2 0x100d6de 0x7fe16fea4340 0x7fe16fb66ff9 0xdb61bb 0xdb66a8 0xdb69f4 0xdb739f 0xdb18bc 0x105e153 0x7fe16fe9c182 0x7fe16fbc947d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"C0DC86"},{"b":"400000","o":"C0D3A2"},{"b":"400000","o":"C0D6DE"},{"b":"7FE16FE94000","o":"10340"},{"b":"7FE16FACF000","o":"97FF9"},{"b":"400000","o":"9B61BB"},{"b":"400000","o":"9B66A8"},{"b":"400000","o":"9B69F4"},{"b":"400000","o":"9B739F"},{"b":"400000","o":"9B18BC"},{"b":"400000","o":"C5E153"},{"b":"7FE16FE94000","o":"8182"},{"b":"7FE16FACF000","o":"FA47D"}],"processInfo":{ "mongodbVersion" : "3.1.4-pre-", "gitVersion" : "cfe82d032b02a68cfd4a60ef4163d985a08c8018", "compiledModules" : [ "enterprise" ], "uname" : { "sysname" : "Linux", "release" : "3.13.0-48-generic", "version" : "#80-Ubuntu SMP Thu Mar 12 11:16:15 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "904DD505A8F66833E0BA932EEE24A1EE8DC84DB5" }, { "b" : "7FFFA24FE000", "elfType" : 3, "buildId" : "341443CC258F9F27709CAA36C8CB321E4BBFFF95" }, { "b" : "7FE172326000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "666B276BD134B0E9579B67D4EE333F2D0FB813CD" }, { "b" : "7FE1720E0000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "EEF6871CB80C0344DF907DD8B0D8C90A0B57D4F0" }, { "b" : "7FE171C73000", "path" : "/usr/lib/x86_64-linux-gnu/libnetsnmpmibs.so.30", "elfType" : 3, "buildId" : "B86A74A738862C45C6E7A4C5E694DD42155710D4" }, { "b" : "7FE171A64000", "path" : "/usr/lib/x86_64-linux-gnu/libsensors.so.4", "elfType" : 3, "buildId" : "859FDBFDD82F0EFDEB44A433D9D8020A232A35E2" }, { "b" : "7FE171860000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "C1AE4CB7195D337A77A3C689051DABAA3980CA0C" }, { "b" : "7FE1715F7000", "path" : "/usr/lib/x86_64-linux-gnu/libnetsnmpagent.so.30", "elfType" : 3, "buildId" : "96C16FDBBA28C6635657AFDBAF0F5A1090072474" }, { "b" : "7FE1713ED000", "path" : "/lib/x86_64-linux-gnu/libwrap.so.0", "elfType" : 3, "buildId" : "54FCBC5B0F994A13A9B0EAD46F23E7DA7F7FE75B" }, { "b" : "7FE171113000", "path" : "/usr/lib/x86_64-linux-gnu/libnetsnmp.so.30", "elfType" : 3, "buildId" : "61AE85EF50A072D671D55B4776383F8365A3FAA7" }, { "b" : "7FE170D38000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "B927879B878D90DD9FF4B15B00E7799AA8E0272F" }, { "b" : "7FE170A32000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "1D76B71E905CB867B27CEF230FCB20F01A3178F5" }, { "b" : "7FE1707D4000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "FF43D0947510134A8A494063A3C1CF3CEBB27791" }, { "b" : "7FE1705CC000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "92FCF41EFE012D6186E31A59AD05BDBB487769AB" }, { "b" : "7FE1702C8000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "19EFDDAB11B3BF5C71570078C59F91CF6592CE9E" }, { "b" : "7FE1700B2000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "8D0AA71411580EE6C08809695C3984769F25725B" }, { "b" : "7FE16FE94000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "9318E8AF0BFBE444731BB0461202EF57F7C39542" }, { "b" : "7FE16FACF000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "30C94DC66A1FE95180C3D68D2B89E576D5AE213C" }, { "b" : "7FE172541000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9F00581AB3C73E3AEA35995A0C50D24D59A01D47" }, { "b" : "7FE16F804000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "00EB4BEB543C549A411AF26E8C2B4BBA806F57AE" }, { "b" : "7FE16F5D5000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "3057CF4B96D55B1CD2C3681B2A1F75279F66F225" }, { "b" : "7FE16F3D1000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "04BF7D9BE17AC2A5F7121B246488932718870207" }, { "b" : "7FE16F1C6000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "86E28C0FCF8D7DAEB9AE77A7C7930F4C2B78A64E" }, { "b" : "7FE16EE3D000", "path" : "/usr/lib/libperl.so.5.18", "elfType" : 3, "buildId" : "FA1AD13C56D51B69C9558106D8DAF14730B4C14F" }, { "b" : "7FE16EC23000", "path" : "/lib/x86_64-linux-gnu/libnsl.so.1", "elfType" : 3, "buildId" : "497315006FCA1547A16E644FB7FEBA8BD2FAB054" }, { "b" : "7FE16EA1F000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "0F03635F97B93D3DACD84F0ED363C56BD266044F" }, { "b" : "7FE16E804000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "616683BCFD8626F176EDA99B6A5D4D2C57996590" }, { "b" : "7FE16E5CB000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "1B0F2710E989E9A581C257DFFDC90085D0E1348A" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x26) [0x100dc86]
mongod(+0xC0D3A2) [0x100d3a2]
mongod(+0xC0D6DE) [0x100d6de]
libpthread.so.0(+0x10340) [0x7fe16fea4340]
libc.so.6(+0x97FF9) [0x7fe16fb66ff9]
mongod(_ZN5mongo3dur11RecoveryJob5writeERNS1_4LastERKNS0_18ParsedJournalEntryE+0x7B) [0xdb61bb]
mongod(_ZN5mongo3dur11RecoveryJob10applyEntryERNS1_4LastERKNS0_18ParsedJournalEntryEbb+0x358) [0xdb66a8]
mongod(_ZN5mongo3dur11RecoveryJob12applyEntriesERKSt6vectorINS0_18ParsedJournalEntryESaIS3_EE+0xE4) [0xdb69f4]
mongod(_ZN5mongo3dur11RecoveryJob14processSectionEPKNS0_11JSectHeaderEPKvjPKNS0_11JSectFooterE+0x6AF) [0xdb739f]
mongod(_ZN5mongo3dur13JournalWriter20_journalWriterThreadEv+0x46C) [0xdb18bc]
mongod(+0xC5E153) [0x105e153]
libpthread.so.0(+0x8182) [0x7fe16fe9c182]
libc.so.6(clone+0x6D) [0x7fe16fbc947d]
----- END BACKTRACE -----