2017-09-15T15:29:28.086-0600 I INDEX [conn5759] build index on: mdb-cdex.SenderProperties properties: { v: 2, unique: true, key: { checksum: 1 }, name: "checksum_1", ns: "mdb-cdex.SenderProperties" } 2017-09-15T15:29:28.086-0600 I INDEX [conn5759] building index using bulk method; build may temporarily use up to 250 megabytes of RAM 2017-09-15T15:29:28.098-0600 I INDEX [conn5759] build index on: mdb-cdex.SenderProperties properties: { v: 2, key: { senderOildexId.companyId: 1, senderOildexId.businessUnit: 1, receiverOildexId.companyId: 1, receiverOildexId.businessUnit: 1, _fts: "text", _ftsx: 1 }, name: "sender_property_name_number_text", ns: "mdb-cdex.SenderProperties", default_language: "none", weights: { propertyName: 1, propertyNumber: 1 }, language_override: "language", textIndexVersion: 3 } 2017-09-15T15:29:28.098-0600 I INDEX [conn5759] building index using bulk method; build may temporarily use up to 250 megabytes of RAM 2017-09-15T15:29:29.925-0600 I INDEX [conn5759] build index done. scanned 154963 total records. 1 secs 2017-09-15T15:29:29.925-0600 I COMMAND [conn5759] command mdb-cdex.$cmd command: createIndexes { createIndexes: "SenderProperties", indexes: [ { key: { checksum: 1 }, name: "checksum_1", ns: "mdb-cdex.SenderProperties", unique: true }, { key: { senderOildexId.companyId: 1, senderOildexId.businessUnit: 1, receiverOildexId.companyId: 1, receiverOildexId.businessUnit: 1, propertyNumber: "text", propertyName: "text" }, name: "sender_property_name_number_text", ns: "mdb-cdex.SenderProperties", default_language: "none" } ] } numYields:0 reslen:113 locks:{ Global: { acquireCount: { r: 3, w: 3 } }, Database: { acquireCount: { w: 2, W: 1 } }, Collection: { acquireCount: { w: 1 } }, Metadata: { acquireCount: { w: 2, W: 1 } }, oplog: { acquireCount: { w: 2 } } } protocol:op_query 1844ms 2017-09-15T15:31:52.435-0600 I - [conn5751] end connection 10.11.5.23:58409 (50 connections now open) 2017-09-15T15:36:50.416-0600 I - [conn5704] Invariant failure amExpr->numChildren() >= prefixEnd src/mongo/db/query/planner_access.cpp 466 2017-09-15T15:36:50.417-0600 I - [conn5704] ***aborting after invariant() failure 2017-09-15T15:36:50.454-0600 F - [conn5704] Got signal: 6 (Aborted). 0x7f5be8704e61 0x7f5be8703f59 0x7f5be870443d 0x7f5be5e5f7e0 0x7f5be5aee495 0x7f5be5aefc75 0x7f5be7998a12 0x7f5be8037fa6 0x7f5be8038313 0x7f5be80387bc 0x7f5be80397ad 0x7f5be80399b8 0x7f5be803b096 0x7f5be804f0dd 0x7f5be7fe68bf 0x7f5be7febb98 0x7f5be7fec7fb 0x7f5be7c13a88 0x7f5be7bea817 0x7f5be7bebbb0 0x7f5be820c82d 0x7f5be7e092b2 0x7f5be7e0b226 0x7f5be7a07ecd 0x7f5be7a0880d 0x7f5be866a6c2 0x7f5be5e57aa1 0x7f5be5ba4bcd ----- BEGIN BACKTRACE ----- {"backtrace":[{"b":"7F5BE7187000","o":"157DE61","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7F5BE7187000","o":"157CF59"},{"b":"7F5BE7187000","o":"157D43D"},{"b":"7F5BE5E50000","o":"F7E0"},{"b":"7F5BE5ABC000","o":"32495","s":"gsignal"},{"b":"7F5BE5ABC000","o":"33C75","s":"abort"},{"b":"7F5BE7187000","o":"811A12","s":"_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j"},{"b":"7F5BE7187000","o":"EB0FA6","s":"_ZN5mongo18QueryPlannerAccess14finishTextNodeEPNS_17QuerySolutionNodeERKNS_10IndexEntryE"},{"b":"7F5BE7187000","o":"EB1313","s":"_ZN5mongo18QueryPlannerAccess14finishLeafNodeEPNS_17QuerySolutionNodeERKNS_10IndexEntryE"},{"b":"7F5BE7187000","o":"EB17BC","s":"_ZN5mongo18QueryPlannerAccess19finishAndOutputLeafEPNS0_17ScanBuildingStateEPSt6vectorIPNS_17QuerySolutionNodeESaIS5_EE"},{"b":"7F5BE7187000","o":"EB27AD","s":"_ZN5mongo18QueryPlannerAccess17processIndexScansERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsEPS6_IPNS_17QuerySolutionNodeESaISG_EE"},{"b":"7F5BE7187000","o":"EB29B8","s":"_ZN5mongo18QueryPlannerAccess15buildIndexedAndERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsE"},{"b":"7F5BE7187000","o":"EB4096","s":"_ZN5mongo18QueryPlannerAccess22buildIndexedDataAccessERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsE"},{"b":"7F5BE7187000","o":"EC80DD","s":"_ZN5mongo12QueryPlanner4planERKNS_14CanonicalQueryERKNS_18QueryPlannerParamsEPSt6vectorIPNS_13QuerySolutionESaIS9_EE"},{"b":"7F5BE7187000","o":"E5F8BF"},{"b":"7F5BE7187000","o":"E64B98","s":"_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS5_EENS_12PlanExecutor11YieldPolicyEm"},{"b":"7F5BE7187000","o":"E657FB","s":"_ZN5mongo15getExecutorFindEPNS_16OperationContextEPNS_10CollectionERKNS_15NamespaceStringESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS8_EENS_12PlanExecutor11YieldPolicyE"},{"b":"7F5BE7187000","o":"A8CA88","s":"_ZN5mongo7FindCmd3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERNS_7BSONObjEiRS8_RNS_14BSONObjBuilderE"},{"b":"7F5BE7187000","o":"A63817","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"7F5BE7187000","o":"A64BB0","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"7F5BE7187000","o":"108582D","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"7F5BE7187000","o":"C822B2"},{"b":"7F5BE7187000","o":"C84226","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"7F5BE7187000","o":"880ECD","s":"_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE"},{"b":"7F5BE7187000","o":"88180D"},{"b":"7F5BE7187000","o":"14E36C2"},{"b":"7F5BE5E50000","o":"7AA1"},{"b":"7F5BE5ABC000","o":"E8BCD","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.4", "gitVersion" : "888390515874a9debd1b6c5d36559ca86b44babd", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "2.6.32-696.1.1.el6.x86_64", "version" : "#1 SMP Tue Apr 11 17:13:24 UTC 2017", "machine" : "x86_64" }, "somap" : [ { "b" : "7F5BE7187000", "elfType" : 3, "buildId" : "8E722BCB27505785E167A3387031F5FCFD801477" }, { "b" : "7FFD69ECA000", "elfType" : 3, "buildId" : "2D6CA602533A1F0F4F46C4C9030B283DC9FA2056" }, { "b" : "7F1D988F8000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "BECFB85A8BC084042D5BF2BA9E66325CE798B659" }, { "b" : "7F1DA1513000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "CBDA444A7109874C5350AE9CEEF3F82F749B347F" }, { "b" : "7F1DA3F0B000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "FDF3A36FFFE08375456D59DA959EAB2FC30B6186" }, { "b" : "7F1DA4107000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "1F7E85410384392BC51FA7324961719A10125F31" }, { "b" : "7F1DA3283000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "8A852AC42F0B64F0F30C760EBBCFA3FE4A228F12" }, { "b" : "7F1DA106D000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "BC7550A8A7C2D706FE4E489058BADC963465DBB7" }, { "b" : "7F1DA3E50000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "85104ECFE42C606B31C2D0D0D2E5DACD3286A341" }, { "b" : "7F1DA3EBC000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "82D26D42DEDBF486501993A5390D79FA7A308D24" }, { "b" : "7F1DA5764000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "3D2B7152131778242EB1D48C7F8E3A1B054FBA89" }, { "b" : "7F1D97C78000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "9A737F8BF10FC99C37CC404D3FC188F6E11FEDD9" }, { "b" : "7F1D98591000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "8D3D6E28DF6EB3752642A7031AAC17D39EA4265D" }, { "b" : "7F1D9B38D000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "57F77704A7F1F4E3689D028D3F9ADD4E77486EC9" }, { "b" : "7F1D97D61000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "CC89B4C8CDCCD32BA610BC72784DC3B7E9BD9E19" }, { "b" : "7F1DA234B000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F1D98940000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "E0C522C589F775C324330BE09CE67DC83950A213" }, { "b" : "7F1D98F3D000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F1DA0923000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "F0BE1166EDCFFB2422B940D601A1BBD89352D80F" }, { "b" : "7F1DA1304000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "B4576BE308DDCF7BC31F7304E4734C3D846D0236" } ] }} mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7f5be8704e61] mongod(+0x157CF59) [0x7f5be8703f59] mongod(+0x157D43D) [0x7f5be870443d] libpthread.so.0(+0xF7E0) [0x7f5be5e5f7e0] libc.so.6(gsignal+0x35) [0x7f5be5aee495] libc.so.6(abort+0x175) [0x7f5be5aefc75] mongod(_ZN5mongo17invariantOKFailedEPKcRKNS_6StatusES1_j+0x0) [0x7f5be7998a12] mongod(_ZN5mongo18QueryPlannerAccess14finishTextNodeEPNS_17QuerySolutionNodeERKNS_10IndexEntryE+0x416) [0x7f5be8037fa6] mongod(_ZN5mongo18QueryPlannerAccess14finishLeafNodeEPNS_17QuerySolutionNodeERKNS_10IndexEntryE+0x263) [0x7f5be8038313] mongod(_ZN5mongo18QueryPlannerAccess19finishAndOutputLeafEPNS0_17ScanBuildingStateEPSt6vectorIPNS_17QuerySolutionNodeESaIS5_EE+0x3C) [0x7f5be80387bc] mongod(_ZN5mongo18QueryPlannerAccess17processIndexScansERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsEPS6_IPNS_17QuerySolutionNodeESaISG_EE+0x41D) [0x7f5be80397ad] mongod(_ZN5mongo18QueryPlannerAccess15buildIndexedAndERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsE+0xB8) [0x7f5be80399b8] mongod(_ZN5mongo18QueryPlannerAccess22buildIndexedDataAccessERKNS_14CanonicalQueryEPNS_15MatchExpressionEbRKSt6vectorINS_10IndexEntryESaIS7_EERKNS_18QueryPlannerParamsE+0x106) [0x7f5be803b096] mongod(_ZN5mongo12QueryPlanner4planERKNS_14CanonicalQueryERKNS_18QueryPlannerParamsEPSt6vectorIPNS_13QuerySolutionESaIS9_EE+0x293D) [0x7f5be804f0dd] mongod(+0xE5F8BF) [0x7f5be7fe68bf] mongod(_ZN5mongo11getExecutorEPNS_16OperationContextEPNS_10CollectionESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS5_EENS_12PlanExecutor11YieldPolicyEm+0x88) [0x7f5be7febb98] mongod(_ZN5mongo15getExecutorFindEPNS_16OperationContextEPNS_10CollectionERKNS_15NamespaceStringESt10unique_ptrINS_14CanonicalQueryESt14default_deleteIS8_EENS_12PlanExecutor11YieldPolicyE+0x8B) [0x7f5be7fec7fb] mongod(_ZN5mongo7FindCmd3runEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERNS_7BSONObjEiRS8_RNS_14BSONObjBuilderE+0xC98) [0x7f5be7c13a88] mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x527) [0x7f5be7bea817] mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x9E0) [0x7f5be7bebbb0] mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x26D) [0x7f5be820c82d] mongod(+0xC822B2) [0x7f5be7e092b2] mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x6D6) [0x7f5be7e0b226] mongod(_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE+0x1ED) [0x7f5be7a07ecd] mongod(+0x88180D) [0x7f5be7a0880d] mongod(+0x14E36C2) [0x7f5be866a6c2] libpthread.so.0(+0x7AA1) [0x7f5be5e57aa1] libc.so.6(clone+0x6D) [0x7f5be5ba4bcd] ----- END BACKTRACE -----