Uploaded image for project: 'Core Server'
  1. Core Server
  2. SERVER-39109

mongod crash: Invariant failure !_exec src/mongo/db/pipeline/document_source_cursor.cpp 295

    • Fully Compatible
    • ALL
    • v4.0, v3.6
    • Hide

      I was writing an aggreggation and after adding one step, server crashed down.

      Aggregation that worked fine is in attached file "aggregation - 01.txt"

      Aggregation that caused the server to crash is in attached file "aggregation - 02.txt"

       

      Also, I've attached "server build info" and "server status" files ( after reboot )

      Show
      I was writing an aggreggation and after adding one step, server crashed down. Aggregation that worked fine is in attached file "aggregation - 01.txt" Aggregation that caused the server to crash is in attached file "aggregation - 02.txt"   Also, I've attached "server build info" and "server status" files ( after reboot )
    • Query 2019-01-28

      Aggregation execution produced following error

      2019-01-21T10:32:29.307+0100 F - [conn3962826] Invariant failure !_exec src/mongo/db/pipeline/document_source_cursor.cpp 295
      2019-01-21T10:32:29.307+0100 F - [conn3962826]
      
      ***aborting after invariant() failure
      
      
      2019-01-21T10:32:29.356+0100 F - [conn3962826] Got signal: 6 (Aborted).
       0x5594974de2e1 0x5594974dd4f9 0x5594974dd9dd 0x7fe845c3d390 0x7fe845897428 0x7fe84589902a 0x559495b06410 0x559496403546 0x5594964035a1 0x5594964059f2 0x5594964092c4 0x5594961c9a5d 0x5594961bf644 0x559496f49d69 0x559495bb5c7d 0x559495bb7909 0x559495bb8841 0x559495ba426a 0x559495bb0d1a 0x559495bab9a7 0x559495baf1f1 0x559496d3a7c2 0x559495ba9b90 0x559495bacd75 0x559495bab0e7 0x559495baba2d 0x559495baf1f1 0x559496d3ad25 0x559497436f54 0x7fe845c336ba 0x7fe84596941d
      ----- BEGIN BACKTRACE -----
      {"backtrace":[\{"b":"5594950F3000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},\{"b":"5594950F3000","o":"23EA4F9"},\{"b":"5594950F3000","o":"23EA9DD"},\{"b":"7FE845C2C000","o":"11390"},\{"b":"7FE845862000","o":"35428","s":"gsignal"},\{"b":"7FE845862000","o":"3702A","s":"abort"},\{"b":"5594950F3000","o":"A13410","s":"_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j"},\{"b":"5594950F3000","o":"1310546","s":"_ZN5mongo20DocumentSourceCursorD1Ev"},\{"b":"5594950F3000","o":"13105A1","s":"_ZN5mongo20DocumentSourceCursorD0Ev"},\{"b":"5594950F3000","o":"13129F2","s":"_ZN5mongo9PipelineD15addCursorSourceEPNS_10CollectionEPNS_8PipelineERKN5boost13intrusive_ptrINS_17ExpressionContextEEESt10unique_ptrINS_12PlanExecutorENSC_7DeleterEENS_11DepsTrackerERKNS_7BSONObjESI_SI_"},\{"b":"5594950F3000","o":"13162C4","s":"_ZN5mongo9PipelineD19prepareCursorSourceEPNS_10CollectionERKNS_15NamespaceStringEPKNS_18AggregationRequestEPNS_8PipelineE"},\{"b":"5594950F3000","o":"10D6A5D","s":"_ZN5mongo12runAggregateEPNS_16OperationContextERKNS_15NamespaceStringERKNS_18AggregationRequestERKNS_7BSONObjERNS_14BSONObjBuilderE"},\{"b":"5594950F3000","o":"10CC644"},\{"b":"5594950F3000","o":"1E56D69","s":"_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE"},\{"b":"5594950F3000","o":"AC2C7D"},\{"b":"5594950F3000","o":"AC4909"},\{"b":"5594950F3000","o":"AC5841","s":"_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE"},\{"b":"5594950F3000","o":"AB126A","s":"_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE"},\{"b":"5594950F3000","o":"ABDD1A","s":"_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE"},\{"b":"5594950F3000","o":"AB89A7","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},\{"b":"5594950F3000","o":"ABC1F1"},\{"b":"5594950F3000","o":"1C477C2","s":"_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE"},\{"b":"5594950F3000","o":"AB6B90","s":"_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE"},\{"b":"5594950F3000","o":"AB9D75","s":"_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE"},\{"b":"5594950F3000","o":"AB80E7","s":"_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE"},\{"b":"5594950F3000","o":"AB8A2D","s":"_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE"},\{"b":"5594950F3000","o":"ABC1F1"},\{"b":"5594950F3000","o":"1C47D25"},\{"b":"5594950F3000","o":"2343F54"},\{"b":"7FE845C2C000","o":"76BA"},\{"b":"7FE845862000","o":"10741D","s":"clone"}],"processInfo":\{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.4.0-141-generic", "version" : "#167-Ubuntu SMP Wed Dec 5 10:40:15 UTC 2018", "machine" : "x86_64" }, "somap" : [ \{ "b" : "5594950F3000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, \{ "b" : "7FFFE1CE1000", "elfType" : 3, "buildId" : "937226BB20C3276922400EE09D268D8E14F8C147" }, \{ "b" : "7FE84703D000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, \{ "b" : "7FE846E22000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, \{ "b" : "7FE8469DD000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, \{ "b" : "7FE846774000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, \{ "b" : "7FE846570000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, \{ "b" : "7FE846368000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, \{ "b" : "7FE84605F000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, \{ "b" : "7FE845E49000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, \{ "b" : "7FE845C2C000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, \{ "b" : "7FE845862000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, \{ "b" : "7FE8472AC000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, \{ "b" : "7FE84562F000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, \{ "b" : "7FE845413000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, \{ "b" : "7FE8451C9000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, \{ "b" : "7FE844FBA000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "D3B183C41F02C7CD18F906AAFD19C69C850F1CEB" }, \{ "b" : "7FE844D69000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "DA0EC53E16B3AC6BDC56EAEFE1BFECDDC395FB2E" }, \{ "b" : "7FE844B4F000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, \{ "b" : "7FE84481F000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, \{ "b" : "7FE8445EC000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, \{ "b" : "7FE8443B6000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, \{ "b" : "7FE844136000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, \{ "b" : "7FE843E64000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, \{ "b" : "7FE843C35000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, \{ "b" : "7FE843A31000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, \{ "b" : "7FE843826000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, \{ "b" : "7FE84360B000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "96BCC7EB28D81B1469EED6F24FC083CBD58577BC" }, \{ "b" : "7FE8433CA000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, \{ "b" : "7FE843166000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, \{ "b" : "7FE842F53000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, \{ "b" : "7FE842D4F000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, \{ "b" : "7FE842B46000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, \{ "b" : "7FE8428BC000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, \{ "b" : "7FE84261A000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, \{ "b" : "7FE8423E7000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, \{ "b" : "7FE8421D1000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, \{ "b" : "7FE841FC9000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, \{ "b" : "7FE841DA0000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, \{ "b" : "7FE841B91000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, \{ "b" : "7FE841946000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, \{ "b" : "7FE841671000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, \{ "b" : "7FE841439000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
       mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x5594974de2e1]
       mongod(+0x23EA4F9) [0x5594974dd4f9]
       mongod(+0x23EA9DD) [0x5594974dd9dd]
       libpthread.so.0(+0x11390) [0x7fe845c3d390]
       libc.so.6(gsignal+0x38) [0x7fe845897428]
       libc.so.6(abort+0x16A) [0x7fe84589902a]
       mongod(_ZN5mongo22invariantFailedWithMsgEPKcRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEES1_j+0x0) [0x559495b06410]
       mongod(_ZN5mongo20DocumentSourceCursorD1Ev+0x216) [0x559496403546]
       mongod(_ZN5mongo20DocumentSourceCursorD0Ev+0x11) [0x5594964035a1]
       mongod(_ZN5mongo9PipelineD15addCursorSourceEPNS_10CollectionEPNS_8PipelineERKN5boost13intrusive_ptrINS_17ExpressionContextEEESt10unique_ptrINS_12PlanExecutorENSC_7DeleterEENS_11DepsTrackerERKNS_7BSONObjESI_SI_+0x4E2) [0x5594964059f2]
       mongod(_ZN5mongo9PipelineD19prepareCursorSourceEPNS_10CollectionERKNS_15NamespaceStringEPKNS_18AggregationRequestEPNS_8PipelineE+0x984) [0x5594964092c4]
       mongod(_ZN5mongo12runAggregateEPNS_16OperationContextERKNS_15NamespaceStringERKNS_18AggregationRequestERKNS_7BSONObjERNS_14BSONObjBuilderE+0x219D) [0x5594961c9a5d]
       mongod(+0x10CC644) [0x5594961bf644]
       mongod(_ZN5mongo12BasicCommand10Invocation3runEPNS_16OperationContextEPNS_19CommandReplyBuilderE+0xD9) [0x559496f49d69]
       mongod(+0xAC2C7D) [0x559495bb5c7d]
       mongod(+0xAC4909) [0x559495bb7909]
       mongod(_ZN5mongo23ServiceEntryPointCommon13handleRequestEPNS_16OperationContextERKNS_7MessageERKNS0_5HooksE+0x3C1) [0x559495bb8841]
       mongod(_ZN5mongo23ServiceEntryPointMongod13handleRequestEPNS_16OperationContextERKNS_7MessageE+0x3A) [0x559495ba426a]
       mongod(_ZN5mongo19ServiceStateMachine15_processMessageENS0_11ThreadGuardE+0xBA) [0x559495bb0d1a]
       mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x97) [0x559495bab9a7]
       mongod(+0xABC1F1) [0x559495baf1f1]
       mongod(_ZN5mongo9transport26ServiceExecutorSynchronous8scheduleESt8functionIFvvEENS0_15ServiceExecutor13ScheduleFlagsENS0_23ServiceExecutorTaskNameE+0x1A2) [0x559496d3a7c2]
       mongod(_ZN5mongo19ServiceStateMachine22_scheduleNextWithGuardENS0_11ThreadGuardENS_9transport15ServiceExecutor13ScheduleFlagsENS2_23ServiceExecutorTaskNameENS0_9OwnershipE+0x150) [0x559495ba9b90]
       mongod(_ZN5mongo19ServiceStateMachine15_sourceCallbackENS_6StatusE+0xB05) [0x559495bacd75]
       mongod(_ZN5mongo19ServiceStateMachine14_sourceMessageENS0_11ThreadGuardE+0x357) [0x559495bab0e7]
       mongod(_ZN5mongo19ServiceStateMachine15_runNextInGuardENS0_11ThreadGuardE+0x11D) [0x559495baba2d]
       mongod(+0xABC1F1) [0x559495baf1f1]
       mongod(+0x1C47D25) [0x559496d3ad25]
       mongod(+0x2343F54) [0x559497436f54]
       libpthread.so.0(+0x76BA) [0x7fe845c336ba]
       libc.so.6(clone+0x6D) [0x7fe84596941d]
      ----- END BACKTRACE -----
      

        1. aggregation - 01.txt
          0.8 kB
          Ivica Hrg
        2. aggregation - 02.txt
          0.8 kB
          Ivica Hrg
        3. server build info.txt
          2 kB
          Ivica Hrg
        4. server status.txt
          64 kB
          Ivica Hrg

            Assignee:
            ian.boros@mongodb.com Ian Boros
            Reporter:
            ihrg Ivica Hrg
            Votes:
            0 Vote for this issue
            Watchers:
            13 Start watching this issue

              Created:
              Updated:
              Resolved: