Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-16022

Assertion hit when running out of disk space

    • Type: Icon: Bug Bug
    • Resolution: Done
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: None
    • Component/s: Storage
    • Labels:
    • Environment:
      $ uname -a
      Linux ip-172-31-18-197 3.10.42-52.145.amzn1.x86_64 #1 SMP Tue Jun 10 23:46:43 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux
    • ALL

      Don't "panic" when we run out of disk space, we should shutdown cleanly instead as mmapv1 does.

      2014-10-13T18:59:54.596+0000 E STORAGE  [conn13] WiredTiger (0) [1413226794:596698][33831000730943a7f0000], cursor.set_value: the WiredTiger library cannot continue; the process must exit and restart
      2014-10-13T18:59:54.596+0000 E STORAGE  [conn13] WiredTiger (0) [1413226794:596720][33831000730943a7f0000], file:collection--7899902353294981286-35$k_1--7899902353294981286-37.wt, cursor.insert: the WiredTiger library cannot continue; the process must exit and restart
      2014-10-13T18:59:54.596+0000 I -        [conn13] Fatal assertion 28519 InternalError -31804: WT_PANIC: WiredTiger library panic
      2014-10-13T18:59:54.596+0000 I -        [conn12] Fatal assertion 28519 InternalError -31804: WT_PANIC: WiredTiger library panic
      2014-10-13T18:59:54.596+0000 I -        [conn15] Fatal assertion 28519 InternalError -31804: WT_PANIC: WiredTiger library panic
      2014-10-13T18:59:54.596+0000 I -        [conn11] Fatal assertion 28519 InternalError -31804: WT_PANIC: WiredTiger library panic
      2014-10-13T18:59:54.596+0000 I -        [conn14] Fatal assertion 28519 InternalError -31804: WT_PANIC: WiredTiger library panic
      2014-10-13T18:59:54.606+0000 I -        [conn11] 
       0xf293a9 0xed51c1 0xebbf0b 0xd6311a 0xd622ab 0xd5b899 0xa6264f 0x9133ee 0x9139c4 0x9027ea 0x902be4 0xa8d467 0xa8ded0 0xa8f313 0xa911a0 0x7f65b0 0xee8481 0x7f3aa5c72f18 0x7f3aa4d84e9d 0
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"B293A9"},{"b":"400000","o":"AD51C1"},{"b":"400000","o":"ABBF0B"},{"b":"400000","o":"96311A"},{"b":"400000","o":"9622AB"},{"b":"400000","o":"95B899"},{"b":"400000","o":"66264F"},{"b":"400000","o":"5133EE"},{"b":"400000","o":"5139C4"},{"b":"400000","o
      ":"5027EA"},{"b":"400000","o":"502BE4"},{"b":"400000","o":"68D467"},{"b":"400000","o":"68DED0"},{"b":"400000","o":"68F313"},{"b":"400000","o":"6911A0"},{"b":"400000","o":"3F65B0"},{"b":"400000","o":"AE8481"},{"b":"7F3AA5C6B000","o":"7F18"},{"b":"7F3AA4CA2000","o":"E2E9D"},{"b":"0",
      "o":"0"}],"processInfo":{ "mongodbVersion" : "2.7.8-pre-", "gitVersion" : "20e6627b01a53077e6f8fbdd223a6ac5c6d1b843", "uname" : { "sysname" : "Linux", "release" : "3.10.42-52.145.amzn1.x86_64", "version" : "#1 SMP Tue Jun 10 23:46:43 UTC 2014", "machine" : "x86_64" }, "somap" : [ {
       "elfType" : 2, "b" : "400000", "buildId" : "FB7BD08CAF4AB6993A0D3417385DB049E68B0EB6" }, { "b" : "7FFF75E86000", "elfType" : 3, "buildId" : "422398C00613A4323A5E0C656ED6BE4BCFF5FD15" }, { "b" : "7F3AA616C000", "path" : "/usr/lib64/libunwind.so.8", "elfType" : 3, "buildId" : "BE5AA
      6DC7578ACED25C4B41E35A0302C66085A20" }, { "b" : "7F3AA5E87000", "path" : "/home/ec2-user/wiredtiger/LOCAL_INSTALL/lib/libwiredtiger-2.3.2.so", "elfType" : 3, "buildId" : "8C4B27551F745BB0A00F13499DDAC8DC7A9F067B" }, { "b" : "7F3AA5C6B000", "path" : "/lib64/libpthread.so.0", "elfTyp
      e" : 3, "buildId" : "3C51D8CB39ED16242013CB77B0125707C6F34406" }, { "b" : "7F3AA5A63000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "92B6FB6A7CF87B575FE6043F95639C1A081E2E2A" }, { "b" : "7F3AA585F000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "F08BBD0
      7F4042BC8D0D314F5E1F7F5D26F028CFB" }, { "b" : "7F3AA555B000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5E2AE3B2E1D3984B4DE32FE73D221D8C425516E5" }, { "b" : "7F3AA525D000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "DFB15C9F2E7C575E1954C19CEFC2
      842DE2C265DB" }, { "b" : "7F3AA5047000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "DB655E06F0F4F7B4EC561BB7E620F5D5BC4F1C54" }, { "b" : "7F3AA4CA2000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "3B97A7F435805785DFD11096836F4E904FFF2599" }, { "b" : "
      7F3AA6385000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "CAD953A4B324B3E3AA1449742558362857E826F2" }, { "b" : "7F3AA4A8C000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "87B4EBF2183C8EA4AB657212203EFFE6340E2F4F" }, { "b" : "7F3AA4887000", "pat
      h" : "/usr/lib64/libsnappy.so.1", "elfType" : 3, "buildId" : "0D547AB53108E1F6C1ABBEC8D23A9A682104C42E" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf293a9]
       mongod(_ZN5mongo10logContextEPKc+0xE1) [0xed51c1]
       mongod(_ZN5mongo23fassertFailedWithStatusEiRKNS_6StatusE+0xAB) [0xebbf0b]
       mongod(_ZN5mongo17WiredTigerSession13releaseCursorEmP11__wt_cursor+0x23A) [0xd6311a]
       mongod(_ZN5mongo16WiredTigerCursorD1Ev+0x2B) [0xd622ab]
       mongod(_ZN5mongo15WiredTigerIndex6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_7DiskLocEb+0x229) [0xd5b899]
       mongod(_ZN5mongo22BtreeBasedAccessMethod6insertEPNS_16OperationContextERKNS_7BSONObjERKNS_7DiskLocERKNS_19InsertDeleteOptionsEPl+0x13F) [0xa6264f]
       mongod(_ZN5mongo12IndexCatalog12_indexRecordEPNS_16OperationContextEPNS_17IndexCatalogEntryERKNS_7BSONObjERKNS_7DiskLocE+0x7E) [0x9133ee]
       mongod(_ZN5mongo12IndexCatalog11indexRecordEPNS_16OperationContextERKNS_7BSONObjERKNS_7DiskLocE+0x54) [0x9139c4]
       mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0xAA) [0x9027ea]
       mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0xD4) [0x902be4]
       mongod(_ZN5mongo14checkAndInsertEPNS_16OperationContextERNS_6Client7ContextEPKcRNS_7BSONObjE+0x127) [0xa8d467]
       mongod(_ZN5mongo11insertMultiEPNS_16OperationContextERNS_6Client7ContextEbPKcRSt6vectorINS_7BSONObjESaIS8_EERNS_5CurOpE+0xB0) [0xa8ded0]
       mongod(_ZN5mongo14receivedInsertEPNS_16OperationContextERNS_7MessageERNS_5CurOpE+0x403) [0xa8f313]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortEb+0x1B20) [0xa911a0]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xE0) [0x7f65b0]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x421) [0xee8481]
       libpthread.so.0(+0x7F18) [0x7f3aa5c72f18]
       libc.so.6(clone+0x6D) [0x7f3aa4d84e9d]
       ??? [0]
      -----  END BACKTRACE  -----
      2014-10-13T18:59:54.606+0000 I -        [conn11] 
      
      ***aborting after fassert() failure
      

            Assignee:
            mark.benvenuto@mongodb.com Mark Benvenuto
            Reporter:
            davide.italiano Davide Italiano
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated:
              Resolved: