Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-21475

Powercycle: mongod cannot recover if server host crashes during a drop database, for mmapv1

    • Type: Icon: Bug Bug
    • Resolution: Duplicate
    • Priority: Icon: Major - P3 Major - P3
    • None
    • Affects Version/s: 3.2.0-rc2
    • Component/s: MMAPv1
    • Labels:
    • ALL
    • Hide

      Using the latest version of the power test script:

      powertest.sh -s <ip addr> -L -N 0 -F ~/mongo -f 10 -e mmapv1 -R -w drop_database.js
      
      Show
      Using the latest version of the power test script: powertest.sh -s <ip addr> -L -N 0 -F ~/mongo -f 10 -e mmapv1 -R -w drop_database.js

      The powercycle test uncovered a problem with mmapv1 during a server crash event, when a drop database is active. mongod recovery fails with the following error:

      2015-11-16T10:39:18.708-0500 I -        [initandlisten] Assertion: 10446:mmap: can't map area of size 0 file: /disk3/mmapv1/data/mmapv1/fsm2_db0drop_database6.ns
      2015-11-16T10:39:18.744-0500 I CONTROL  [initandlisten]
       0x12ccd92 0x126b0f4 0x1257ca8 0x1257d5c 0x100bde4 0x1005869 0xff15cd 0xfd49f1 0x101081a 0x1013bae 0xad1593 0x984654 0x93fda4 0x7f691b074af5 0x9817ed
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"ECCD92"},{"b":"400000","o":"E6B0F4"},{"b":"400000","o":"E57CA8"},{"b":"400000","o":"E57D5C"},{"b":"400000","o":"C0BDE4"},{"b":"400000","o":"C05869"},{"b":"400000","o":"BF15CD"},{"b":"400000","o":"BD49F1"},{"b":"400000","o":"C1081A"},{"b":"400000","o":"C13BAE"},{"b":"400000","o":"6D1593"},{"b":"400000","o":"584654"},{"b":"400000","o":"53FDA4"},{"b":"7F691B053000","o":"21AF5"},{"b":"400000","o":"5817ED"}],"processInfo":{ "mongodbVersion" : "3.2.0-rc2", "gitVersion" : "8a3acb42742182c5e314636041c2df368232bbc5", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-229.20.1.el7.x86_64", "version" : "#1 SMP Tue Nov 3 19:10:07 UTC 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "93EB348A8DE4174EE33D1F97B15C6E13A3D1C641" }, { "b" : "7FFE566D6000", "elfType" : 3, "buildId" : "248B82562C046037E264A35BBF5233A2E4A8FBE3" }, { "b" : "7F691C642000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "BB96EE99138B19FECDAB55E80A1728B648ECAD50" }, { "b" : "7F691C25B000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "B154203FB7C05AEE29D5D6F6C000305191209FE4" }, { "b" : "7F691C053000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "7376A07360DC57189A8F92B20AA4AA1CAEA80551" }, { "b" : "7F691BE4F000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "4DFEE4EA9AE8FDD4C71BD4CCC0727222F19DF810" }, { "b" : "7F691BB48000", "path" : "/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "405EACD649720B8668FFBBA197CBF030A7EF6296" }, { "b" : "7F691B846000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "A1AA62B29765BE03A36BF927B047EEEF8696EEC6" }, { "b" : "7F691B630000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "5D3D7256AE68BCFF41E312A24825ED80ECA88A73" }, { "b" : "7F691B414000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "12F30315D4F4A2FE58B1977405C8B5515861E66B" }, { "b" : "7F691B053000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "C31FFE7942BFD77B2FCA8F9BD5709D387A86D3BC" }, { "b" : "7F691C8AF000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9866E1D2BA61EBB4CE4F009FACDAACC24EF3B804" }, { "b" : "7F691AE07000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "34672D541C8C9C5C1C25CB4F3F332CC9D3E604AD" }, { "b" : "7F691AB24000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "45CB7F6CD322F5B55FF8B635F7EC1578631CCAEA" }, { "b" : "7F691A920000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "3A1166709F88740C49E060731832E3FAD2DFB66B" }, { "b" : "7F691A6EE000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "23A2D854538903E2B84EF0882046DD95522C8B59" }, { "b" : "7F691A4D8000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "E45643F27F3B3E960F3691AFC6EC27A98EF7B46B" }, { "b" : "7F691A2C9000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "F4A3D5E7E23F871751CA8F250421F8CF83447AD2" }, { "b" : "7F691A0C5000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F6919EAB000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "AC596E865AF0D14B10F7B707F47D2031AD6D68DC" }, { "b" : "7F6919C86000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "82FF6B18E1E42825CC2D060F969479AD4AF2F62C" }, { "b" : "7F6919A25000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "298B19C64B19995F2AA4DA7B852E90BA5302F630" }, { "b" : "7F6919800000", "path" : "/lib64/liblzma.so.5", "elfType" : 3, "buildId" : "218D03D1F6CF1A099A4D467B5E8ECF4F2BF45750" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x32) [0x12ccd92]
       mongod(_ZN5mongo10logContextEPKc+0x134) [0x126b0f4]
       mongod(_ZN5mongo11msgassertedEiPKc+0x88) [0x1257ca8]
       mongod(+0xE57D5C) [0x1257d5c]
       mongod(_ZN5mongo16MemoryMappedFile3mapEPKcRyi+0xAC4) [0x100bde4]
       mongod(_ZN5mongo16MemoryMappedFile14mapWithOptionsEPKci+0x59) [0x1005869]
       mongod(_ZN5mongo17DurableMappedFile4openERKSsb+0x5D) [0xff15cd]
       mongod(_ZN5mongo14NamespaceIndex4initEPNS_16OperationContextE+0x71) [0xfd49f1]
       mongod(_ZN5mongo26MMAPV1DatabaseCatalogEntryC1EPNS_16OperationContextENS_10StringDataES3_bb+0x17A) [0x101081a]
       mongod(_ZN5mongo12MMAPV1Engine23getDatabaseCatalogEntryEPNS_16OperationContextENS_10StringDataE+0x14E) [0x1013bae]
       mongod(_ZN5mongo14DatabaseHolder6openDbEPNS_16OperationContextENS_10StringDataEPb+0x133) [0xad1593]
       mongod(_ZN5mongo13initAndListenEi+0xD84) [0x984654]
       mongod(main+0x154) [0x93fda4]
       libc.so.6(__libc_start_main+0xF5) [0x7f691b074af5]
       mongod(+0x5817ED) [0x9817ed]
      -----  END BACKTRACE  -----
      2015-11-16T10:39:18.744-0500 W STORAGE  [initandlisten] database /disk3/mmapv1/data/mmapv1 fsm2_db0drop_database6 could not be opened due to DBException 10446: mmap: can't map area of size 0 file: /disk3/mmapv1/data/mmapv1/fsm2_db0drop_database6.ns
      2015-11-16T10:39:18.750-0500 I STORAGE  [initandlisten] exception in initAndListen: 10446 mmap: can't map area of size 0 file: /disk3/mmapv1/data/mmapv1/fsm2_db0drop_database6.ns, terminating
      

            Assignee:
            Unassigned Unassigned
            Reporter:
            jonathan.abrahams Jonathan Abrahams
            Votes:
            0 Vote for this issue
            Watchers:
            6 Start watching this issue

              Created:
              Updated:
              Resolved: