Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-15711

mongod hits dbexception in groupCommit

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • 2.8.0-rc0
    • Affects Version/s: 2.7.8
    • Component/s: Concurrency, Storage
    • Labels:
    • Environment:
      $ uname -a
      Linux ip-172-31-18-197 3.10.42-52.145.amzn1.x86_64 #1 SMP Tue Jun 10 23:46:43 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux
    • ALL
    • Hide

      run sysbench

      Show
      run sysbench

      Got while running sysbench (insert phase), with CLL enabled.
      I extracted the relevant bits from the log, everything else is just message reporting inserts.

      2014-10-16T23:24:03.329+0000 I QUERY    [conn157] insert sbtest.sbtest5 ninserted:1000 keyUpdates:0 numYields:0  104ms
      2014-10-16T23:24:03.330+0000 I QUERY    [conn155] insert sbtest.sbtest3 ninserted:1000 keyUpdates:0 numYields:0  106ms
      2014-10-16T23:24:03.352+0000 I QUERY    [conn156] insert sbtest.sbtest4 ninserted:1000 keyUpdates:0 numYields:0  103ms
      2014-10-16T23:24:05.341+0000 I -        [conn153] warning assertion failure _intents.size() < 2000000 src/mongo/db/storage/mmap_v1/dur_commitjob.h 116
      2014-10-16T23:24:05.347+0000 I -        [conn153] 
       0xf1fbd9 0xecb9c1 0xeb2219 0xd1e4bf 0xd199ba 0xd2b027 0xd2b18d 0x89c506 0xa8c13e 0xa8cb10 0xa8df53 0xa8fda0 0x7f12c0 0xedecb1 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB2219"},{"b":"400000","o":"91E4BF"},{"b":"400000","o":"9199BA"},{"b":"400000","o":"92B027"},{"b":"400000","o":"92B18D"},{"b":"400000","o":"49C506"},{"b":"400000","o":"68C13E"},{"b":"400000","o
      ":"68CB10"},{"b":"400000","o":"68DF53"},{"b":"400000","o":"68FDA0"},{"b":"400000","o":"3F12C0"},{"b":"400000","o":"ADECB1"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093
      b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FF
      F81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" :
       3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2
      AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F
      5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo9wassertedEPKcS1_j+0x199) [0xeb2219]
       mongod(_ZN5mongo3dur9CommitJob4noteEPvi+0x3BF) [0xd1e4bf]
       mongod(_ZN5mongo3dur11DurableImpl10writingPtrEPvj+0xA) [0xd199ba]
       mongod(_ZN5mongo15DurRecoveryUnit14publishChangesEv+0x77) [0xd2b027]
       mongod(_ZN5mongo15DurRecoveryUnit16commitUnitOfWorkEv+0x4D) [0xd2b18d]
       mongod(_ZN5mongo15WriteUnitOfWork6commitEv+0x16) [0x89c506]
       mongod(_ZN5mongo14checkAndInsertEPNS_16OperationContextERNS_6Client7ContextEPKcRNS_7BSONObjE+0x15E) [0xa8c13e]
       mongod(_ZN5mongo11insertMultiEPNS_16OperationContextERNS_6Client7ContextEbPKcRSt6vectorINS_7BSONObjESaIS8_EERNS_5CurOpE+0xB0) [0xa8cb10]
       mongod(_ZN5mongo14receivedInsertEPNS_16OperationContextERNS_7MessageERNS_5CurOpE+0x403) [0xa8df53]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0x1AE0) [0xa8fda0]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7f12c0]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x421) [0xedecb1]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      2014-10-16T23:24:06.644+0000 I -        [journal] warning assertion failure a <= 256*1024*1024 src/mongo/util/alignedbuilder.cpp 106
      2014-10-16T23:24:06.648+0000 I -        [journal] 
       0xf1fbd9 0xecb9c1 0xeb2219 0xeb184d 0xd24ea1 0xd25314 0xd1c25c 0xf6c6b4 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB2219"},{"b":"400000","o":"AB184D"},{"b":"400000","o":"924EA1"},{"b":"400000","o":"925314"},{"b":"400000","o":"91C25C"},{"b":"400000","o":"B6C6B4"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3
      AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "
      somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FFF81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId"
       : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5
      E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, 
      { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "p
      ath" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo9wassertedEPKcS1_j+0x199) [0xeb2219]
       mongod(_ZN5mongo14AlignedBuilder14growReallocateEj+0x4D) [0xeb184d]
       mongod(+0x924EA1) [0xd24ea1]
       mongod(_ZN5mongo3dur13PREPLOGBUFFERERNS0_11JSectHeaderERNS_14AlignedBuilderE+0x204) [0xd25314]
       mongod(+0x91C25C) [0xd1c25c]
       mongod(+0xB6C6B4) [0xf6c6b4]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      2014-10-16T23:24:14.969+0000 I QUERY    [conn158] insert sbtest.sbtest7 ninserted:1000 keyUpdates:0 numYields:0  8857ms
      

      [...]

      2014-10-16T23:24:28.728+0000 W STORAGE  [conn154] DR102 too much data written uncommitted 314.577MB
      

      [...]

      2014-10-16T23:24:45.493+0000 I QUERY    [conn159] insert sbtest.sbtest8 ninserted:1000 keyUpdates:0 numYields:0  103ms
      2014-10-16T23:24:47.030+0000 I -        [conn153] warning assertion failure _intents.size() < 2000000 src/mongo/db/storage/mmap_v1/dur_commitjob.h 116
      2014-10-16T23:24:47.036+0000 I -        [conn153] 
       0xf1fbd9 0xecb9c1 0xeb2219 0xd1e4bf 0xd199ba 0xd2b027 0xd2b18d 0x89c506 0xa8c13e 0xa8cb10 0xa8df53 0xa8fda0 0x7f12c0 0xedecb1 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB2219"},{"b":"400000","o":"91E4BF"},{"b":"400000","o":"9199BA"},{"b":"400000","o":"92B027"},{"b":"400000","o":"92B18D"},{"b":"400000","o":"49C506"},{"b":"400000","o":"68C13E"},{"b":"400000","o
      ":"68CB10"},{"b":"400000","o":"68DF53"},{"b":"400000","o":"68FDA0"},{"b":"400000","o":"3F12C0"},{"b":"400000","o":"ADECB1"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093
      b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FF
      F81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" :
       3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2
      AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F
      5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo9wassertedEPKcS1_j+0x199) [0xeb2219]
       mongod(_ZN5mongo3dur9CommitJob4noteEPvi+0x3BF) [0xd1e4bf]
       mongod(_ZN5mongo3dur11DurableImpl10writingPtrEPvj+0xA) [0xd199ba]
       mongod(_ZN5mongo15DurRecoveryUnit14publishChangesEv+0x77) [0xd2b027]
       mongod(_ZN5mongo15DurRecoveryUnit16commitUnitOfWorkEv+0x4D) [0xd2b18d]
       mongod(_ZN5mongo15WriteUnitOfWork6commitEv+0x16) [0x89c506]
       mongod(_ZN5mongo14checkAndInsertEPNS_16OperationContextERNS_6Client7ContextEPKcRNS_7BSONObjE+0x15E) [0xa8c13e]
       mongod(_ZN5mongo11insertMultiEPNS_16OperationContextERNS_6Client7ContextEbPKcRSt6vectorINS_7BSONObjESaIS8_EERNS_5CurOpE+0xB0) [0xa8cb10]
       mongod(_ZN5mongo14receivedInsertEPNS_16OperationContextERNS_7MessageERNS_5CurOpE+0x403) [0xa8df53]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0x1AE0) [0xa8fda0]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7f12c0]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x421) [0xedecb1]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      2014-10-16T23:24:47.353+0000 I QUERY    [conn156] insert sbtest.sbtest4 ninserted:1000 keyUpdates:0 numYields:0  106ms
      2014-10-16T23:24:47.366+0000 I QUERY    [conn155] insert sbtest.sbtest3 ninserted:1000 keyUpdates:0 numYields:0  108ms
      2014-10-16T23:24:47.379+0000 I QUERY    [conn159] insert sbtest.sbtest8 ninserted:1000 keyUpdates:0 numYields:0  107ms
      2014-10-16T23:24:51.533+0000 I QUERY    [conn157] insert sbtest.sbtest5 ninserted:1000 keyUpdates:0 numYields:0  4206ms
      2014-10-16T23:24:51.607+0000 I QUERY    [conn158] insert sbtest.sbtest7 ninserted:1000 keyUpdates:0 numYields:0  4234ms
      2014-10-16T23:24:52.591+0000 I -        [conn153] warning assertion failure _intents.size() < 2000000 src/mongo/db/storage/mmap_v1/dur_commitjob.h 116
      2014-10-16T23:24:52.598+0000 I -        [conn153] 
       0xf1fbd9 0xecb9c1 0xeb2219 0xd1e4bf 0xd199ba 0xd2b027 0xd2b18d 0x89c506 0xa8c13e 0xa8cb10 0xa8df53 0xa8fda0 0x7f12c0 0xedecb1 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB2219"},{"b":"400000","o":"91E4BF"},{"b":"400000","o":"9199BA"},{"b":"400000","o":"92B027"},{"b":"400000","o":"92B18D"},{"b":"400000","o":"49C506"},{"b":"400000","o":"68C13E"},{"b":"400000","o
      ":"68CB10"},{"b":"400000","o":"68DF53"},{"b":"400000","o":"68FDA0"},{"b":"400000","o":"3F12C0"},{"b":"400000","o":"ADECB1"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093
      b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FF
      F81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" :
       3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2
      AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F
      5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo9wassertedEPKcS1_j+0x199) [0xeb2219]
       mongod(_ZN5mongo3dur9CommitJob4noteEPvi+0x3BF) [0xd1e4bf]
       mongod(_ZN5mongo3dur11DurableImpl10writingPtrEPvj+0xA) [0xd199ba]
       mongod(_ZN5mongo15DurRecoveryUnit14publishChangesEv+0x77) [0xd2b027]
       mongod(_ZN5mongo15DurRecoveryUnit16commitUnitOfWorkEv+0x4D) [0xd2b18d]
       mongod(_ZN5mongo15WriteUnitOfWork6commitEv+0x16) [0x89c506]
       mongod(_ZN5mongo14checkAndInsertEPNS_16OperationContextERNS_6Client7ContextEPKcRNS_7BSONObjE+0x15E) [0xa8c13e]
       mongod(_ZN5mongo11insertMultiEPNS_16OperationContextERNS_6Client7ContextEbPKcRSt6vectorINS_7BSONObjESaIS8_EERNS_5CurOpE+0xB0) [0xa8cb10]
       mongod(_ZN5mongo14receivedInsertEPNS_16OperationContextERNS_7MessageERNS_5CurOpE+0x403) [0xa8df53]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0x1AE0) [0xa8fda0]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7f12c0]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x421) [0xedecb1]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      

      [...]

      2014-10-16T23:25:18.165+0000 I QUERY    [conn155] insert sbtest.sbtest3 ninserted:1000 keyUpdates:0 numYields:0  30773ms
      2014-10-16T23:25:18.829+0000 I -        [journal] warning assertion failure a <= 256*1024*1024 src/mongo/util/alignedbuilder.cpp 106
      2014-10-16T23:25:18.832+0000 I -        [journal] 
       0xf1fbd9 0xecb9c1 0xeb2219 0xeb184d 0xd24ea1 0xd25314 0xd1c25c 0xf6c6b4 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB2219"},{"b":"400000","o":"AB184D"},{"b":"400000","o":"924EA1"},{"b":"400000","o":"925314"},{"b":"400000","o":"91C25C"},{"b":"400000","o":"B6C6B4"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3
      AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "
      somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FFF81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId"
       : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5
      E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, 
      { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "p
      ath" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo9wassertedEPKcS1_j+0x199) [0xeb2219]
       mongod(_ZN5mongo14AlignedBuilder14growReallocateEj+0x4D) [0xeb184d]
       mongod(+0x924EA1) [0xd24ea1]
       mongod(_ZN5mongo3dur13PREPLOGBUFFERERNS0_11JSectHeaderERNS_14AlignedBuilderE+0x204) [0xd25314]
       mongod(+0x91C25C) [0xd1c25c]
       mongod(+0xB6C6B4) [0xf6c6b4]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      2014-10-16T23:25:19.273+0000 I -        [journal] Assertion failure a <= 512*1024*1024 src/mongo/util/alignedbuilder.cpp 107
      2014-10-16T23:25:19.277+0000 I -        [journal] 
       0xf1fbd9 0xecb9c1 0xeb306f 0xeb186d 0xd24ea1 0xd25314 0xd1c25c 0xf6c6b4 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"ACB9C1"},{"b":"400000","o":"AB306F"},{"b":"400000","o":"AB186D"},{"b":"400000","o":"924EA1"},{"b":"400000","o":"925314"},{"b":"400000","o":"91C25C"},{"b":"400000","o":"B6C6B4"},{"b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3
      AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "
      somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FFF81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId"
       : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD07F4042BC8D0D314F5
      E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, 
      { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "7F72B4CDA000", "p
      ath" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xecb9c1]
       mongod(_ZN5mongo12verifyFailedEPKcS1_j+0xEF) [0xeb306f]
       mongod(_ZN5mongo14AlignedBuilder14growReallocateEj+0x6D) [0xeb186d]
       mongod(+0x924EA1) [0xd24ea1]
       mongod(_ZN5mongo3dur13PREPLOGBUFFERERNS0_11JSectHeaderERNS_14AlignedBuilderE+0x204) [0xd25314]
       mongod(+0x91C25C) [0xd1c25c]
       mongod(+0xB6C6B4) [0xf6c6b4]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      

      [...]
      And then there's the fatal one:

      2014-10-16T23:25:19.277+0000 I JOURNAL  [journal] dbexception in groupCommit causing immediate shutdown: 0 assertion src/mongo/util/alignedbuilder.cpp:107
      2014-10-16T23:25:19.277+0000 F COMMANDS [journal] gc1
      2014-10-16T23:25:19.280+0000 F -        [journal] Got signal: 6 (Aborted).
      
       0xf1fbd9 0xf1f792 0xf1fb16 0x7f72b3b28fc0 0x7f72b3b28f49 0x7f72b3b2a348 0xa8a4fc 0xd1cc91 0xf6c6b4 0x7f72b4ac5f18 0x7f72b3bd7e9d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B1FBD9"},{"b":"400000","o":"B1F792"},{"b":"400000","o":"B1FB16"},{"b":"7F72B3AF5000","o":"33FC0"},{"b":"7F72B3AF5000","o":"33F49"},{"b":"7F72B3AF5000","o":"35348"},{"b":"400000","o":"68A4FC"},{"b":"400000","o":"91CC91"},{"b":"400000","o":"B6C6B4"},{
      "b":"7F72B4ABE000","o":"7F18"},{"b":"7F72B3AF5000","o":"E2E9D"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "a7ad32c3fe56d5e6244487d1093b76e280e3522e", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23
      :46:43 UTC 2014", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "DBAE580A976581FBE7A77AABDF01B6741FEC9091" }, { "b" : "7FFF81EFE000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F72B4ABE000", "path" : "/lib64
      /libpthread.so.0", "elfType" : 3, "buildId" : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F72B48B6000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F72B46B2000", "path" : "/lib64/libdl.so.2", "elfType"
       : 3, "buildId" : "F08BBD07F4042BC8D0D314F5E1F7F5D26F028CFB" }, { "b" : "7F72B43AE000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F72B40B0000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DF
      B15C9F2E7C575E1954C19CEFC2842DE2C265DB" }, { "b" : "7F72B3E9A000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7F72B3AF5000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F
      4E904FFF2599" }, { "b" : "7F72B4CDA000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf1fbd9]
       mongod(+0xB1F792) [0xf1f792]
       mongod(+0xB1FB16) [0xf1fb16]
       libc.so.6(+0x33FC0) [0x7f72b3b28fc0]
       libc.so.6(gsignal+0x39) [0x7f72b3b28f49]
       libc.so.6(abort+0x148) [0x7f72b3b2a348]
       mongod(_ZN5mongo10mongoAbortEPKc+0x7C) [0xa8a4fc]
       mongod(+0x91CC91) [0xd1cc91]
       mongod(+0xB6C6B4) [0xf6c6b4]
       libpthread.so.0(+0x7F18) [0x7f72b4ac5f18]
       libc.so.6(clone+0x6D) [0x7f72b3bd7e9d]
      -----  END BACKTRACE  -----
      

            Assignee:
            kaloian.manassiev@mongodb.com Kaloian Manassiev
            Reporter:
            davide.italiano Davide Italiano
            Votes:
            0 Vote for this issue
            Watchers:
            8 Start watching this issue

              Created:
              Updated:
              Resolved: