Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-22749

Better error logging for failures when writing an audit record to the audit log

    XMLWordPrintable

    Details

    • Type: Improvement
    • Status: Open
    • Priority: Major - P3
    • Resolution: Unresolved
    • Affects Version/s: None
    • Fix Version/s: Backlog
    • Component/s: Logging
    • Labels:
    • Case:

      Description

      When the filesystem for audit logging is full, the mongod crashes with the following stacktrace:

      2016-02-18T18:42:03.394-0030 F -        [conn1] Got signal: 6 (Aborted).
       
       0xfbe589 0xfbdc02 0xfbdfb6 0x334f232920 0x334f2328a5 0x334f234085 0xb08a16 0xafba6c 0x8efc9e 0x9dcb04 0x9dda8d 0x9de79b 0xbea8ba 0xac1c10 0x80775d 0xf6f92b 0x334fa07851 0x334f2e890d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[{"b":"400000","o":"BBE589"},{"b":"400000","o":"BBDC02"},{"b":"400000","o":"BBDFB6"},{"b":"334F200000","o":"32920"},{"b":"334F200000","o":"328A5"},{"b":"334F200000","o":"34085"},{"b":"400000","o":"708A16"},{"b":"400000","o":"6FBA6C"},{"b":"400000","o":"4EFC9E"},{"b":"400000","o":"5DCB04"},{"b":"400000","o":"5DDA8D"},{"b":"400000","o":"5DE79B"},{"b":"400000","o":"7EA8BA"},{"b":"400000","o":"6C1C10"},{"b":"400000","o":"40775D"},{"b":"400000","o":"B6F92B"},{"b":"334FA00000","o":"7851"},{"b":"334F200000","o":"E890D"}],"processInfo":{ "mongodbVersion" : "3.0.4", "gitVersion" : "0481c958daeb2969800511e7475dc66986fa9ed5 modules: enterprise", "uname" : { "sysname" : "Linux", "release" : "2.6.32-358.14.1.el6.x86_64", "version" : "#1 SMP Mon Jun 17 15:54:20 EDT 2013", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "7C955CCB3FB890113097592FAFCBCF575226A625" }, { "b" : "7FFF094FF000", "elfType" : 3, "buildId" : "96123C074A81DD1E6A6D276A7D0BC2F9AD758ED9" }, { "path" : "/usr/lib64/libnetsnmpagent.so.20", "elfType" : 3, "buildId" : "D3FE2D70D98AE4377019B1391B42CC50C3DA3B45" }, { "path" : "/usr/lib64/libnetsnmphelpers.so.20", "elfType" : 3, "buildId" : "222C6CC004E2AE579571649AA5D7F6D2B91EE838" }, { "path" : "/usr/lib64/libnetsnmpmibs.so.20", "elfType" : 3, "buildId" : "8D134DDA29BE89EABCED7345963955DD93B6A2F3" }, { "path" : "/usr/lib64/libnetsnmp.so.20", "elfType" : 3, "buildId" : "C5D43C058301254FDB4B79BCD46738EC0DD4BDEA" }, { "path" : "/usr/lib64/libsasl2.so.2", "elfType" : 3, "buildId" : "9E79FB1A0DDCEC497B5D8AFD86CC378ACF8E354A" }, { "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "76A3DEEB6876CBED69A57D3EBC1E2AFBCA84EC76" }, { "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "3FF31EFC5E0E5CFC4BFDAE19F3DE3AD55DA766CD" }, { "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "C953DA0BE16117BDE323D1E021FC74878825C935" }, { "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "0DA1E4A1752A229A5C833E6319C16920BFBC2C5A" }, { "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "756DBE5D1255F42B13E0659E3DD791D34A91465A" }, { "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "A38407EE35545AEA5CF08FE4CAA8B66E5909B6F3" }, { "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "5864321409B6F3ED0CA97C4FD19A5FA1199F2D00" }, { "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "1686CCDAE5F8CED5A251E40074F55EFDF1688B75" }, { "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "A2E6E550A824EBC44AE5487B290A00923DB37761" }, { "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "479A85D69A50CAB26A1F0576C58E2DB3A7BE2440" }, { "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "DE7D207393D303AF233E6AD4D1E8A8314843422A" }, { "path" : "/lib64/libwrap.so.0", "elfType" : 3, "buildId" : "8C0C7CAB7F028E4592A8581EB2122FBECAB26B97" }, { "b" : "7EFCF80D8000", "path" : "/usr/lib64/perl5/CORE/libperl.so", "elfType" : 3, "buildId" : "9DBE3B60B9BFA01EF3D1DCB02B6EA3B55017AAD1" }, { "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "6309B69A475D35D4E93D31DB3A8DDAF5100075C8" }, { "path" : "/lib64/libnsl.so.1", "elfType" : 3, "buildId" : "FE814F77668083C2F523969809B9F84B5D17C196" }, { "path" : "/lib64/libcrypt.so.1", "elfType" : 3, "buildId" : "89AEEA4C9A2F8A4595BD387408EA4BB3B421F4CD" }, { "path" : "/lib64/libutil.so.1", "elfType" : 3, "buildId" : "3651AC7B011C57166F65C46B0666E6BC8B9F35CD" }, { "path" : "/usr/lib64/librpm.so.1", "elfType" : 3, "buildId" : "2D7DB4D3DE53CFB1DA72B3A2FC2BD39513720689" }, { "path" : "/usr/lib64/librpmio.so.1", "elfType" : 3, "buildId" : "C092936F05C17E1F48EFCCAC98F33133D1078755" }, { "path" : "/lib64/libpopt.so.0", "elfType" : 3, "buildId" : "E7B49911F1136073DD7DC58E8118CD9A4FBE2A19" }, { "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "D053BB4FF0C2FC983842F81598813B9B931AD0D1" }, { "path" : "/usr/lib64/libsensors.so.4", "elfType" : 3, "buildId" : "6855E5BF5B3634C15F01B1043BD892D727EE3C08" }, { "b" : "7EFCF69EE000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "A4D98A7611F0CDBEA8A4FEDF36BDA25E772F387E" }, { "b" : "7EFCF53C2000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "190D45F6743DEF9DF8169D65801D4575B01825BD" }, { "b" : "7EFCF71BE000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "F1A67CF54F08AFBEE05E316BAFD9EF168F258800" }, { "b" : "7EFCF67B2000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "DAE2A7E4E8B37D43EF6839FF5D8E012AFCF21A69" }, { "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "8A8734DC37305D8CC2EF8F8C3E5EA03171DB07EC" }, { "path" : "/lib64/libfreebl3.so", "elfType" : 3, "buildId" : "486F43B831ACA76E47622608C4F67419288E764A" }, { "path" : "/usr/lib64/libnss3.so", "elfType" : 3, "buildId" : "BD98678CA72469CBF04F3C23D5CE05453B32F95C" }, { "path" : "/lib64/libbz2.so.1", "elfType" : 3, "buildId" : "1250B1D041DD7552F0C870BB188DC3A34DF2651D" }, { "path" : "/usr/lib64/libelf.so.1", "elfType" : 3, "buildId" : "17B1D7503CCD3C5DBC7284A7ADFDFFD84B0B3329" }, { "path" : "/usr/lib64/liblzma.so.0", "elfType" : 3, "buildId" : "6FF9BAEEEE9DDEEF2DFA5CBD36147A75891C0AD4" }, { "path" : "/usr/lib64/liblua-5.1.so", "elfType" : 3, "buildId" : "6BDB4E1990D6EBA12A5C8D39A7650DB8798BF568" }, { "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "BAD5C71361DADF259B6E306A49E6F47F24AEA3DC" }, { "path" : "/lib64/libcap.so.2", "elfType" : 3, "buildId" : "A436538388F1F25113FDA834CA2EED524EFA17D6" }, { "path" : "/lib64/libacl.so.1", "elfType" : 3, "buildId" : "26CC708AC7C0FC1797A2340C024F0ADD0CE054D8" }, { "path" : "/lib64/libdb-4.7.so", "elfType" : 3, "buildId" : "4CF43E1CEBBD1469A97108247CE0680B0141D8D7" }, { "path" : "/usr/lib64/libnssutil3.so", "elfType" : 3, "buildId" : "51D858074A434536895BF899660121A7CEECC73C" }, { "path" : "/lib64/libplc4.so", "elfType" : 3, "buildId" : "308D0303DEF062BA04F77A02F92FA926CC131BC1" }, { "path" : "/lib64/libplds4.so", "elfType" : 3, "buildId" : "7F0A628ACC8BFA47ADF8D003984F36F40AA67A89" }, { "path" : "/lib64/libnspr4.so", "elfType" : 3, "buildId" : "BA46A3309139A437E0D4EBF972F1C3DECDC0D315" }, { "path" : "/lib64/libattr.so.1", "elfType" : 3, "buildId" : "8EF0683858704EF173AB11B1E27076F37F82B7B6" }, { "b" : "7F3045111000", "path" : "/usr/lib64/gconv/ISO8859-1.so", "elfType" : 3, "buildId" : "B610D171ABED12D2B5DCAABAF10A1A5DF806D68E" }, { "b" : "7F3044F0C000", "path" : "/usr/lib64/sasl2/liblogin.so", "elfType" : 3, "buildId" : "161930A21E55FFADF2FAB366184D2F398D922E92" }, { "b" : "7F3044CFF000", "path" : "/usr/lib64/sasl2/libdigestmd5.so", "elfType" : 3, "buildId" : "59CF90146D760E586F0360B194141D1AAE15427F" }, { "b" : "7F3044AF9000", "path" : "/usr/lib64/sasl2/libsasldb.so", "elfType" : 3, "buildId" : "20A9C9C04FA83BF4A04AA68833D8774B463BF22E" }, { "b" : "7F30448F3000", "path" : "/usr/lib64/sasl2/libcrammd5.so", "elfType" : 3, "buildId" : "81EBAAB9FB79C6FBF16113946E8E51F59D22DCE6" }, { "b" : "7F30446EE000", "path" : "/usr/lib64/sasl2/libplain.so", "elfType" : 3, "buildId" : "22E4D23A991A44E27DC0D22DB519BC1E22DE3B89" }, { "b" : "7F30444E6000", "path" : "/usr/lib64/sasl2/libgssapiv2.so", "elfType" : 3, "buildId" : "E57DA6F11C14E01FE883007596479A0AA9F6D0AD" }, { "b" : "7F30442E1000", "path" : "/usr/lib64/sasl2/libanonymous.so", "elfType" : 3, "buildId" : "DEA21FA91ABB7116E372416CBAEC076AECB0B94C" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xfbe589]
       mongod(+0xBBDC02) [0xfbdc02]
       mongod(+0xBBDFB6) [0xfbdfb6]
       libc.so.6(+0x32920) [0x334f232920]
       libc.so.6(gsignal+0x35) [0x334f2328a5]
       libc.so.6(abort+0x175) [0x334f234085]
       mongod(_ZN5mongo6logger9LogDomainINS_5audit10AuditEventEE6appendERKS3_+0x146) [0xb08a16]
       mongod(_ZN5mongo5audit17logAuthenticationEPNS_11ClientBasicERKNS_10StringDataERKNS_8UserNameENS_10ErrorCodes5ErrorE+0x28C) [0xafba6c]
       mongod(+0x4EFC9E) [0x8efc9e]
       mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x34) [0x9dcb04]
       mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC1D) [0x9dda8d]
       mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9de79b]
       mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x77A) [0xbea8ba]
       mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xB10) [0xac1c10]
       mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x80775d]
       mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf6f92b]
       libpthread.so.0(+0x7851) [0x334fa07851]
       libc.so.6(clone+0x6D) [0x334f2e890d]
      -----  END BACKTRACE  -----
      

      which is not super clear. It would be nice if there was a more informative message saying the audit logging filesystem is full.

        Attachments

          Issue Links

            Activity

              People

              Assignee:
              backlog-server-security Backlog - Security Team
              Reporter:
              joanna.cheng Joanna Cheng
              Participants:
              Votes:
              2 Vote for this issue
              Watchers:
              12 Start watching this issue

                Dates

                Created:
                Updated: